Actual source code: fdda.c
petsc-dev 2014-02-02
2: #include <petsc-private/dmdaimpl.h> /*I "petscdmda.h" I*/
3: #include <petscmat.h>
4: #include <petsc-private/matimpl.h>
6: extern PetscErrorCode DMCreateColoring_DA_1d_MPIAIJ(DM,ISColoringType,ISColoring*);
7: extern PetscErrorCode DMCreateColoring_DA_2d_MPIAIJ(DM,ISColoringType,ISColoring*);
8: extern PetscErrorCode DMCreateColoring_DA_2d_5pt_MPIAIJ(DM,ISColoringType,ISColoring*);
9: extern PetscErrorCode DMCreateColoring_DA_3d_MPIAIJ(DM,ISColoringType,ISColoring*);
11: /*
12: For ghost i that may be negative or greater than the upper bound this
13: maps it into the 0:m-1 range using periodicity
14: */
15: #define SetInRange(i,m) ((i < 0) ? m+i : ((i >= m) ? i-m : i))
19: static PetscErrorCode DMDASetBlockFills_Private(const PetscInt *dfill,PetscInt w,PetscInt **rfill)
20: {
22: PetscInt i,j,nz,*fill;
25: if (!dfill) return(0);
27: /* count number nonzeros */
28: nz = 0;
29: for (i=0; i<w; i++) {
30: for (j=0; j<w; j++) {
31: if (dfill[w*i+j]) nz++;
32: }
33: }
34: PetscMalloc1((nz + w + 1),&fill);
35: /* construct modified CSR storage of nonzero structure */
36: /* fill[0 -- w] marks starts of each row of column indices (and end of last row)
37: so fill[1] - fill[0] gives number of nonzeros in first row etc */
38: nz = w + 1;
39: for (i=0; i<w; i++) {
40: fill[i] = nz;
41: for (j=0; j<w; j++) {
42: if (dfill[w*i+j]) {
43: fill[nz] = j;
44: nz++;
45: }
46: }
47: }
48: fill[w] = nz;
50: *rfill = fill;
51: return(0);
52: }
56: /*@
57: DMDASetBlockFills - Sets the fill pattern in each block for a multi-component problem
58: of the matrix returned by DMCreateMatrix().
60: Logically Collective on DMDA
62: Input Parameter:
63: + da - the distributed array
64: . dfill - the fill pattern in the diagonal block (may be NULL, means use dense block)
65: - ofill - the fill pattern in the off-diagonal blocks
68: Level: developer
70: Notes: This only makes sense when you are doing multicomponent problems but using the
71: MPIAIJ matrix format
73: The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
74: representing coupling and 0 entries for missing coupling. For example
75: $ dfill[9] = {1, 0, 0,
76: $ 1, 1, 0,
77: $ 0, 1, 1}
78: means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with
79: itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the
80: diagonal block).
82: DMDASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
83: can be represented in the dfill, ofill format
85: Contributed by Glenn Hammond
87: .seealso DMCreateMatrix(), DMDASetGetMatrix(), DMSetMatrixPreallocateOnly()
89: @*/
90: PetscErrorCode DMDASetBlockFills(DM da,const PetscInt *dfill,const PetscInt *ofill)
91: {
92: DM_DA *dd = (DM_DA*)da->data;
94: PetscInt i,k,cnt = 1;
97: DMDASetBlockFills_Private(dfill,dd->w,&dd->dfill);
98: DMDASetBlockFills_Private(ofill,dd->w,&dd->ofill);
100: /* ofillcount tracks the columns of ofill that have any nonzero in thems; the value in each location is the number of
101: columns to the left with any nonzeros in them plus 1 */
102: PetscCalloc1(dd->w,&dd->ofillcols);
103: for (i=0; i<dd->w; i++) {
104: for (k=dd->ofill[i]; k<dd->ofill[i+1]; k++) dd->ofillcols[dd->ofill[k]] = 1;
105: }
106: for (i=0; i<dd->w; i++) {
107: if (dd->ofillcols[i]) {
108: dd->ofillcols[i] = cnt++;
109: }
110: }
111: return(0);
112: }
117: PetscErrorCode DMCreateColoring_DA(DM da,ISColoringType ctype,ISColoring *coloring)
118: {
119: PetscErrorCode ierr;
120: PetscInt dim,m,n,p,nc;
121: DMDABoundaryType bx,by,bz;
122: MPI_Comm comm;
123: PetscMPIInt size;
124: PetscBool isBAIJ;
125: DM_DA *dd = (DM_DA*)da->data;
128: /*
129: m
130: ------------------------------------------------------
131: | |
132: | |
133: | ---------------------- |
134: | | | |
135: n | yn | | |
136: | | | |
137: | .--------------------- |
138: | (xs,ys) xn |
139: | . |
140: | (gxs,gys) |
141: | |
142: -----------------------------------------------------
143: */
145: /*
146: nc - number of components per grid point
147: col - number of colors needed in one direction for single component problem
149: */
150: DMDAGetInfo(da,&dim,0,0,0,&m,&n,&p,&nc,0,&bx,&by,&bz,0);
152: PetscObjectGetComm((PetscObject)da,&comm);
153: MPI_Comm_size(comm,&size);
154: if (ctype == IS_COLORING_GHOSTED) {
155: if (size == 1) {
156: ctype = IS_COLORING_GLOBAL;
157: } else if (dim > 1) {
158: if ((m==1 && bx == DMDA_BOUNDARY_PERIODIC) || (n==1 && by == DMDA_BOUNDARY_PERIODIC) || (p==1 && bz == DMDA_BOUNDARY_PERIODIC)) {
159: SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"IS_COLORING_GHOSTED cannot be used for periodic boundary condition having both ends of the domain on the same process");
160: }
161: }
162: }
164: /* Tell the DMDA it has 1 degree of freedom per grid point so that the coloring for BAIJ
165: matrices is for the blocks, not the individual matrix elements */
166: PetscStrcmp(da->mattype,MATBAIJ,&isBAIJ);
167: if (!isBAIJ) {PetscStrcmp(da->mattype,MATMPIBAIJ,&isBAIJ);}
168: if (!isBAIJ) {PetscStrcmp(da->mattype,MATSEQBAIJ,&isBAIJ);}
169: if (isBAIJ) {
170: dd->w = 1;
171: dd->xs = dd->xs/nc;
172: dd->xe = dd->xe/nc;
173: dd->Xs = dd->Xs/nc;
174: dd->Xe = dd->Xe/nc;
175: }
177: /*
178: We do not provide a getcoloring function in the DMDA operations because
179: the basic DMDA does not know about matrices. We think of DMDA as being more
180: more low-level then matrices.
181: */
182: if (dim == 1) {
183: DMCreateColoring_DA_1d_MPIAIJ(da,ctype,coloring);
184: } else if (dim == 2) {
185: DMCreateColoring_DA_2d_MPIAIJ(da,ctype,coloring);
186: } else if (dim == 3) {
187: DMCreateColoring_DA_3d_MPIAIJ(da,ctype,coloring);
188: } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Not done for %D dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
189: if (isBAIJ) {
190: dd->w = nc;
191: dd->xs = dd->xs*nc;
192: dd->xe = dd->xe*nc;
193: dd->Xs = dd->Xs*nc;
194: dd->Xe = dd->Xe*nc;
195: }
196: return(0);
197: }
199: /* ---------------------------------------------------------------------------------*/
203: PetscErrorCode DMCreateColoring_DA_2d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
204: {
205: PetscErrorCode ierr;
206: PetscInt xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,M,N,dim,s,k,nc,col;
207: PetscInt ncolors;
208: MPI_Comm comm;
209: DMDABoundaryType bx,by;
210: DMDAStencilType st;
211: ISColoringValue *colors;
212: DM_DA *dd = (DM_DA*)da->data;
215: /*
216: nc - number of components per grid point
217: col - number of colors needed in one direction for single component problem
219: */
220: DMDAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&bx,&by,0,&st);
221: col = 2*s + 1;
222: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
223: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
224: PetscObjectGetComm((PetscObject)da,&comm);
226: /* special case as taught to us by Paul Hovland */
227: if (st == DMDA_STENCIL_STAR && s == 1) {
228: DMCreateColoring_DA_2d_5pt_MPIAIJ(da,ctype,coloring);
229: } else {
231: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X (%d) is divisible\n\
232: by 2*stencil_width + 1 (%d)\n", m, col);
233: if (by == DMDA_BOUNDARY_PERIODIC && (n % col)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y (%d) is divisible\n\
234: by 2*stencil_width + 1 (%d)\n", n, col);
235: if (ctype == IS_COLORING_GLOBAL) {
236: if (!dd->localcoloring) {
237: PetscMalloc1(nc*nx*ny,&colors);
238: ii = 0;
239: for (j=ys; j<ys+ny; j++) {
240: for (i=xs; i<xs+nx; i++) {
241: for (k=0; k<nc; k++) {
242: colors[ii++] = k + nc*((i % col) + col*(j % col));
243: }
244: }
245: }
246: ncolors = nc + nc*(col-1 + col*(col-1));
247: ISColoringCreate(comm,ncolors,nc*nx*ny,colors,&dd->localcoloring);
248: }
249: *coloring = dd->localcoloring;
250: } else if (ctype == IS_COLORING_GHOSTED) {
251: if (!dd->ghostedcoloring) {
252: PetscMalloc1(nc*gnx*gny,&colors);
253: ii = 0;
254: for (j=gys; j<gys+gny; j++) {
255: for (i=gxs; i<gxs+gnx; i++) {
256: for (k=0; k<nc; k++) {
257: /* the complicated stuff is to handle periodic boundaries */
258: colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
259: }
260: }
261: }
262: ncolors = nc + nc*(col - 1 + col*(col-1));
263: ISColoringCreate(comm,ncolors,nc*gnx*gny,colors,&dd->ghostedcoloring);
264: /* PetscIntView(ncolors,(PetscInt*)colors,0); */
266: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
267: }
268: *coloring = dd->ghostedcoloring;
269: } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
270: }
271: ISColoringReference(*coloring);
272: return(0);
273: }
275: /* ---------------------------------------------------------------------------------*/
279: PetscErrorCode DMCreateColoring_DA_3d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
280: {
281: PetscErrorCode ierr;
282: PetscInt xs,ys,nx,ny,i,j,gxs,gys,gnx,gny,m,n,p,dim,s,k,nc,col,zs,gzs,ii,l,nz,gnz,M,N,P;
283: PetscInt ncolors;
284: MPI_Comm comm;
285: DMDABoundaryType bx,by,bz;
286: DMDAStencilType st;
287: ISColoringValue *colors;
288: DM_DA *dd = (DM_DA*)da->data;
291: /*
292: nc - number of components per grid point
293: col - number of colors needed in one direction for single component problem
295: */
296: DMDAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&bx,&by,&bz,&st);
297: col = 2*s + 1;
298: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
299: by 2*stencil_width + 1\n");
300: if (by == DMDA_BOUNDARY_PERIODIC && (n % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
301: by 2*stencil_width + 1\n");
302: if (bz == DMDA_BOUNDARY_PERIODIC && (p % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
303: by 2*stencil_width + 1\n");
305: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
306: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
307: PetscObjectGetComm((PetscObject)da,&comm);
309: /* create the coloring */
310: if (ctype == IS_COLORING_GLOBAL) {
311: if (!dd->localcoloring) {
312: PetscMalloc1(nc*nx*ny*nz,&colors);
313: ii = 0;
314: for (k=zs; k<zs+nz; k++) {
315: for (j=ys; j<ys+ny; j++) {
316: for (i=xs; i<xs+nx; i++) {
317: for (l=0; l<nc; l++) {
318: colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
319: }
320: }
321: }
322: }
323: ncolors = nc + nc*(col-1 + col*(col-1)+ col*col*(col-1));
324: ISColoringCreate(comm,ncolors,nc*nx*ny*nz,colors,&dd->localcoloring);
325: }
326: *coloring = dd->localcoloring;
327: } else if (ctype == IS_COLORING_GHOSTED) {
328: if (!dd->ghostedcoloring) {
329: PetscMalloc1(nc*gnx*gny*gnz,&colors);
330: ii = 0;
331: for (k=gzs; k<gzs+gnz; k++) {
332: for (j=gys; j<gys+gny; j++) {
333: for (i=gxs; i<gxs+gnx; i++) {
334: for (l=0; l<nc; l++) {
335: /* the complicated stuff is to handle periodic boundaries */
336: colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
337: }
338: }
339: }
340: }
341: ncolors = nc + nc*(col-1 + col*(col-1)+ col*col*(col-1));
342: ISColoringCreate(comm,ncolors,nc*gnx*gny*gnz,colors,&dd->ghostedcoloring);
343: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
344: }
345: *coloring = dd->ghostedcoloring;
346: } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
347: ISColoringReference(*coloring);
348: return(0);
349: }
351: /* ---------------------------------------------------------------------------------*/
355: PetscErrorCode DMCreateColoring_DA_1d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
356: {
357: PetscErrorCode ierr;
358: PetscInt xs,nx,i,i1,gxs,gnx,l,m,M,dim,s,nc,col;
359: PetscInt ncolors;
360: MPI_Comm comm;
361: DMDABoundaryType bx;
362: ISColoringValue *colors;
363: DM_DA *dd = (DM_DA*)da->data;
366: /*
367: nc - number of components per grid point
368: col - number of colors needed in one direction for single component problem
370: */
371: DMDAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&bx,0,0,0);
372: col = 2*s + 1;
374: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points %d is divisible\n\
375: by 2*stencil_width + 1 %d\n",(int)m,(int)col);
377: DMDAGetCorners(da,&xs,0,0,&nx,0,0);
378: DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
379: PetscObjectGetComm((PetscObject)da,&comm);
381: /* create the coloring */
382: if (ctype == IS_COLORING_GLOBAL) {
383: if (!dd->localcoloring) {
384: PetscMalloc1(nc*nx,&colors);
385: if (dd->ofillcols) {
386: PetscInt tc = 0;
387: for (i=0; i<nc; i++) tc += (PetscInt) (dd->ofillcols[i] > 0);
388: i1 = 0;
389: for (i=xs; i<xs+nx; i++) {
390: for (l=0; l<nc; l++) {
391: if (dd->ofillcols[l] && (i % col)) {
392: colors[i1++] = nc - 1 + tc*((i % col) - 1) + dd->ofillcols[l];
393: } else {
394: colors[i1++] = l;
395: }
396: }
397: }
398: ncolors = nc + 2*s*tc;
399: } else {
400: i1 = 0;
401: for (i=xs; i<xs+nx; i++) {
402: for (l=0; l<nc; l++) {
403: colors[i1++] = l + nc*(i % col);
404: }
405: }
406: ncolors = nc + nc*(col-1);
407: }
408: ISColoringCreate(comm,ncolors,nc*nx,colors,&dd->localcoloring);
409: }
410: *coloring = dd->localcoloring;
411: } else if (ctype == IS_COLORING_GHOSTED) {
412: if (!dd->ghostedcoloring) {
413: PetscMalloc1(nc*gnx,&colors);
414: i1 = 0;
415: for (i=gxs; i<gxs+gnx; i++) {
416: for (l=0; l<nc; l++) {
417: /* the complicated stuff is to handle periodic boundaries */
418: colors[i1++] = l + nc*(SetInRange(i,m) % col);
419: }
420: }
421: ncolors = nc + nc*(col-1);
422: ISColoringCreate(comm,ncolors,nc*gnx,colors,&dd->ghostedcoloring);
423: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
424: }
425: *coloring = dd->ghostedcoloring;
426: } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
427: ISColoringReference(*coloring);
428: return(0);
429: }
433: PetscErrorCode DMCreateColoring_DA_2d_5pt_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
434: {
435: PetscErrorCode ierr;
436: PetscInt xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,dim,s,k,nc;
437: PetscInt ncolors;
438: MPI_Comm comm;
439: DMDABoundaryType bx,by;
440: ISColoringValue *colors;
441: DM_DA *dd = (DM_DA*)da->data;
444: /*
445: nc - number of components per grid point
446: col - number of colors needed in one direction for single component problem
448: */
449: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,0);
450: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
451: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
452: PetscObjectGetComm((PetscObject)da,&comm);
454: if (bx == DMDA_BOUNDARY_PERIODIC && (m % 5)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible by 5\n");
455: if (by == DMDA_BOUNDARY_PERIODIC && (n % 5)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible by 5\n");
457: /* create the coloring */
458: if (ctype == IS_COLORING_GLOBAL) {
459: if (!dd->localcoloring) {
460: PetscMalloc1(nc*nx*ny,&colors);
461: ii = 0;
462: for (j=ys; j<ys+ny; j++) {
463: for (i=xs; i<xs+nx; i++) {
464: for (k=0; k<nc; k++) {
465: colors[ii++] = k + nc*((3*j+i) % 5);
466: }
467: }
468: }
469: ncolors = 5*nc;
470: ISColoringCreate(comm,ncolors,nc*nx*ny,colors,&dd->localcoloring);
471: }
472: *coloring = dd->localcoloring;
473: } else if (ctype == IS_COLORING_GHOSTED) {
474: if (!dd->ghostedcoloring) {
475: PetscMalloc1(nc*gnx*gny,&colors);
476: ii = 0;
477: for (j=gys; j<gys+gny; j++) {
478: for (i=gxs; i<gxs+gnx; i++) {
479: for (k=0; k<nc; k++) {
480: colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
481: }
482: }
483: }
484: ncolors = 5*nc;
485: ISColoringCreate(comm,ncolors,nc*gnx*gny,colors,&dd->ghostedcoloring);
486: ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
487: }
488: *coloring = dd->ghostedcoloring;
489: } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
490: return(0);
491: }
493: /* =========================================================================== */
494: extern PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ(DM,Mat);
495: extern PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ_Fill(DM,Mat);
496: extern PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ(DM,Mat);
497: extern PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ_Fill(DM,Mat);
498: extern PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ(DM,Mat);
499: extern PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ_Fill(DM,Mat);
500: extern PetscErrorCode DMCreateMatrix_DA_2d_MPIBAIJ(DM,Mat);
501: extern PetscErrorCode DMCreateMatrix_DA_3d_MPIBAIJ(DM,Mat);
502: extern PetscErrorCode DMCreateMatrix_DA_2d_MPISBAIJ(DM,Mat);
503: extern PetscErrorCode DMCreateMatrix_DA_3d_MPISBAIJ(DM,Mat);
507: /*@C
508: MatSetupDM - Sets the DMDA that is to be used by the HYPRE_StructMatrix PETSc matrix
510: Logically Collective on Mat
512: Input Parameters:
513: + mat - the matrix
514: - da - the da
516: Level: intermediate
518: @*/
519: PetscErrorCode MatSetupDM(Mat mat,DM da)
520: {
526: PetscTryMethod(mat,"MatSetupDM_C",(Mat,DM),(mat,da));
527: return(0);
528: }
532: PetscErrorCode MatView_MPI_DA(Mat A,PetscViewer viewer)
533: {
534: DM da;
535: PetscErrorCode ierr;
536: const char *prefix;
537: Mat Anatural;
538: AO ao;
539: PetscInt rstart,rend,*petsc,i;
540: IS is;
541: MPI_Comm comm;
542: PetscViewerFormat format;
545: /* Check whether we are just printing info, in which case MatView() already viewed everything we wanted to view */
546: PetscViewerGetFormat(viewer,&format);
547: if (format == PETSC_VIEWER_ASCII_INFO || format == PETSC_VIEWER_ASCII_INFO_DETAIL) return(0);
549: PetscObjectGetComm((PetscObject)A,&comm);
550: MatGetDM(A, &da);
551: if (!da) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONG,"Matrix not generated from a DMDA");
553: DMDAGetAO(da,&ao);
554: MatGetOwnershipRange(A,&rstart,&rend);
555: PetscMalloc1((rend-rstart),&petsc);
556: for (i=rstart; i<rend; i++) petsc[i-rstart] = i;
557: AOApplicationToPetsc(ao,rend-rstart,petsc);
558: ISCreateGeneral(comm,rend-rstart,petsc,PETSC_OWN_POINTER,&is);
560: /* call viewer on natural ordering */
561: MatGetSubMatrix(A,is,is,MAT_INITIAL_MATRIX,&Anatural);
562: ISDestroy(&is);
563: PetscObjectGetOptionsPrefix((PetscObject)A,&prefix);
564: PetscObjectSetOptionsPrefix((PetscObject)Anatural,prefix);
565: PetscObjectSetName((PetscObject)Anatural,((PetscObject)A)->name);
566: MatView(Anatural,viewer);
567: MatDestroy(&Anatural);
568: return(0);
569: }
573: PetscErrorCode MatLoad_MPI_DA(Mat A,PetscViewer viewer)
574: {
575: DM da;
577: Mat Anatural,Aapp;
578: AO ao;
579: PetscInt rstart,rend,*app,i;
580: IS is;
581: MPI_Comm comm;
584: PetscObjectGetComm((PetscObject)A,&comm);
585: MatGetDM(A, &da);
586: if (!da) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONG,"Matrix not generated from a DMDA");
588: /* Load the matrix in natural ordering */
589: MatCreate(PetscObjectComm((PetscObject)A),&Anatural);
590: MatSetType(Anatural,((PetscObject)A)->type_name);
591: MatSetSizes(Anatural,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);
592: MatLoad(Anatural,viewer);
594: /* Map natural ordering to application ordering and create IS */
595: DMDAGetAO(da,&ao);
596: MatGetOwnershipRange(Anatural,&rstart,&rend);
597: PetscMalloc1((rend-rstart),&app);
598: for (i=rstart; i<rend; i++) app[i-rstart] = i;
599: AOPetscToApplication(ao,rend-rstart,app);
600: ISCreateGeneral(comm,rend-rstart,app,PETSC_OWN_POINTER,&is);
602: /* Do permutation and replace header */
603: MatGetSubMatrix(Anatural,is,is,MAT_INITIAL_MATRIX,&Aapp);
604: MatHeaderReplace(A,Aapp);
605: ISDestroy(&is);
606: MatDestroy(&Anatural);
607: return(0);
608: }
612: PetscErrorCode DMCreateMatrix_DA(DM da, Mat *J)
613: {
615: PetscInt dim,dof,nx,ny,nz,dims[3],starts[3],M,N,P;
616: Mat A;
617: MPI_Comm comm;
618: MatType Atype;
619: PetscSection section, sectionGlobal;
620: void (*aij)(void)=NULL,(*baij)(void)=NULL,(*sbaij)(void)=NULL;
621: MatType mtype;
622: PetscMPIInt size;
623: DM_DA *dd = (DM_DA*)da->data;
626: MatInitializePackage();
627: mtype = da->mattype;
629: DMGetDefaultSection(da, §ion);
630: if (section) {
631: PetscInt bs = -1;
632: PetscInt localSize;
633: PetscBool isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isSymmetric;
635: DMGetDefaultGlobalSection(da, §ionGlobal);
636: PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);
637: MatCreate(PetscObjectComm((PetscObject)da), J);
638: MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);
639: MatSetType(*J, mtype);
640: MatSetFromOptions(*J);
641: PetscStrcmp(mtype, MATSHELL, &isShell);
642: PetscStrcmp(mtype, MATBAIJ, &isBlock);
643: PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);
644: PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);
645: PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);
646: PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);
647: PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);
648: /* Check for symmetric storage */
649: isSymmetric = (PetscBool) (isSymBlock || isSymSeqBlock || isSymMPIBlock);
650: if (isSymmetric) {
651: MatSetOption(*J, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);
652: }
653: if (!isShell) {
654: PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal;
656: if (bs < 0) {
657: if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
658: PetscInt pStart, pEnd, p, dof;
660: PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);
661: for (p = pStart; p < pEnd; ++p) {
662: PetscSectionGetDof(sectionGlobal, p, &dof);
663: if (dof) {
664: bs = dof;
665: break;
666: }
667: }
668: } else {
669: bs = 1;
670: }
671: /* Must have same blocksize on all procs (some might have no points) */
672: bsLocal = bs;
673: MPI_Allreduce(&bsLocal, &bs, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)da));
674: }
675: PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);
676: /* DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix); */
677: PetscFree4(dnz, onz, dnzu, onzu);
678: }
679: }
680: /*
681: m
682: ------------------------------------------------------
683: | |
684: | |
685: | ---------------------- |
686: | | | |
687: n | ny | | |
688: | | | |
689: | .--------------------- |
690: | (xs,ys) nx |
691: | . |
692: | (gxs,gys) |
693: | |
694: -----------------------------------------------------
695: */
697: /*
698: nc - number of components per grid point
699: col - number of colors needed in one direction for single component problem
701: */
702: M = dd->M;
703: N = dd->N;
704: P = dd->P;
705: dim = dd->dim;
706: dof = dd->w;
707: /* DMDAGetInfo(da,&dim,&M,&N,&P,0,0,0,&dof,0,0,0,0,0); */
708: DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);
709: PetscObjectGetComm((PetscObject)da,&comm);
710: MatCreate(comm,&A);
711: MatSetSizes(A,dof*nx*ny*nz,dof*nx*ny*nz,dof*M*N*P,dof*M*N*P);
712: MatSetType(A,mtype);
713: MatSetDM(A,da);
714: MatSetFromOptions(A);
715: MatGetType(A,&Atype);
716: /*
717: We do not provide a getmatrix function in the DMDA operations because
718: the basic DMDA does not know about matrices. We think of DMDA as being more
719: more low-level than matrices. This is kind of cheating but, cause sometimes
720: we think of DMDA has higher level than matrices.
722: We could switch based on Atype (or mtype), but we do not since the
723: specialized setting routines depend only the particular preallocation
724: details of the matrix, not the type itself.
725: */
726: PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
727: if (!aij) {
728: PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
729: }
730: if (!aij) {
731: PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
732: if (!baij) {
733: PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
734: }
735: if (!baij) {
736: PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
737: if (!sbaij) {
738: PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
739: }
740: }
741: }
742: if (aij) {
743: if (dim == 1) {
744: if (dd->ofill) {
745: DMCreateMatrix_DA_1d_MPIAIJ_Fill(da,A);
746: } else {
747: DMCreateMatrix_DA_1d_MPIAIJ(da,A);
748: }
749: } else if (dim == 2) {
750: if (dd->ofill) {
751: DMCreateMatrix_DA_2d_MPIAIJ_Fill(da,A);
752: } else {
753: DMCreateMatrix_DA_2d_MPIAIJ(da,A);
754: }
755: } else if (dim == 3) {
756: if (dd->ofill) {
757: DMCreateMatrix_DA_3d_MPIAIJ_Fill(da,A);
758: } else {
759: DMCreateMatrix_DA_3d_MPIAIJ(da,A);
760: }
761: }
762: } else if (baij) {
763: if (dim == 2) {
764: DMCreateMatrix_DA_2d_MPIBAIJ(da,A);
765: } else if (dim == 3) {
766: DMCreateMatrix_DA_3d_MPIBAIJ(da,A);
767: } else SETERRQ3(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s in %D dimension! Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype,dim);
768: } else if (sbaij) {
769: if (dim == 2) {
770: DMCreateMatrix_DA_2d_MPISBAIJ(da,A);
771: } else if (dim == 3) {
772: DMCreateMatrix_DA_3d_MPISBAIJ(da,A);
773: } else SETERRQ3(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s in %D dimension! Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype,dim);
774: } else {
775: ISLocalToGlobalMapping ltog,ltogb;
776: DMGetLocalToGlobalMapping(da,<og);
777: DMGetLocalToGlobalMappingBlock(da,<ogb);
778: MatSetUp(A);
779: MatSetLocalToGlobalMapping(A,ltog,ltog);
780: MatSetLocalToGlobalMappingBlock(A,ltogb,ltogb);
781: }
782: DMDAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
783: MatSetStencil(A,dim,dims,starts,dof);
784: MatSetDM(A,da);
785: MPI_Comm_size(comm,&size);
786: if (size > 1) {
787: /* change viewer to display matrix in natural ordering */
788: MatShellSetOperation(A, MATOP_VIEW, (void (*)(void))MatView_MPI_DA);
789: MatShellSetOperation(A, MATOP_LOAD, (void (*)(void))MatLoad_MPI_DA);
790: }
791: *J = A;
792: return(0);
793: }
795: /* ---------------------------------------------------------------------------------*/
798: PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ(DM da,Mat J)
799: {
800: PetscErrorCode ierr;
801: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny,m,n,dim,s,*cols = NULL,k,nc,*rows = NULL,col,cnt,l,p;
802: PetscInt lstart,lend,pstart,pend,*dnz,*onz;
803: MPI_Comm comm;
804: PetscScalar *values;
805: DMDABoundaryType bx,by;
806: ISLocalToGlobalMapping ltog,ltogb;
807: DMDAStencilType st;
810: /*
811: nc - number of components per grid point
812: col - number of colors needed in one direction for single component problem
814: */
815: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
816: col = 2*s + 1;
817: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
818: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
819: PetscObjectGetComm((PetscObject)da,&comm);
821: PetscMalloc2(nc,&rows,col*col*nc*nc,&cols);
822: DMGetLocalToGlobalMapping(da,<og);
823: DMGetLocalToGlobalMappingBlock(da,<ogb);
825: /* determine the matrix preallocation information */
826: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
827: for (i=xs; i<xs+nx; i++) {
829: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
830: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
832: for (j=ys; j<ys+ny; j++) {
833: slot = i - gxs + gnx*(j - gys);
835: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
836: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
838: cnt = 0;
839: for (k=0; k<nc; k++) {
840: for (l=lstart; l<lend+1; l++) {
841: for (p=pstart; p<pend+1; p++) {
842: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
843: cols[cnt++] = k + nc*(slot + gnx*l + p);
844: }
845: }
846: }
847: rows[k] = k + nc*(slot);
848: }
849: MatPreallocateSetLocal(ltog,nc,rows,ltog,cnt,cols,dnz,onz);
850: }
851: }
852: MatSetBlockSize(J,nc);
853: MatSeqAIJSetPreallocation(J,0,dnz);
854: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
855: MatPreallocateFinalize(dnz,onz);
857: MatSetLocalToGlobalMapping(J,ltog,ltog);
858: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
860: /*
861: For each node in the grid: we get the neighbors in the local (on processor ordering
862: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
863: PETSc ordering.
864: */
865: if (!da->prealloc_only) {
866: PetscCalloc1(col*col*nc*nc,&values);
867: for (i=xs; i<xs+nx; i++) {
869: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
870: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
872: for (j=ys; j<ys+ny; j++) {
873: slot = i - gxs + gnx*(j - gys);
875: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
876: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
878: cnt = 0;
879: for (k=0; k<nc; k++) {
880: for (l=lstart; l<lend+1; l++) {
881: for (p=pstart; p<pend+1; p++) {
882: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
883: cols[cnt++] = k + nc*(slot + gnx*l + p);
884: }
885: }
886: }
887: rows[k] = k + nc*(slot);
888: }
889: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
890: }
891: }
892: PetscFree(values);
893: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
894: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
895: }
896: PetscFree2(rows,cols);
897: return(0);
898: }
902: PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ_Fill(DM da,Mat J)
903: {
904: PetscErrorCode ierr;
905: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
906: PetscInt m,n,dim,s,*cols,k,nc,row,col,cnt,l,p;
907: PetscInt lstart,lend,pstart,pend,*dnz,*onz;
908: DM_DA *dd = (DM_DA*)da->data;
909: PetscInt ifill_col,*ofill = dd->ofill, *dfill = dd->dfill;
910: MPI_Comm comm;
911: PetscScalar *values;
912: DMDABoundaryType bx,by;
913: ISLocalToGlobalMapping ltog,ltogb;
914: DMDAStencilType st;
917: /*
918: nc - number of components per grid point
919: col - number of colors needed in one direction for single component problem
921: */
922: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
923: col = 2*s + 1;
924: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
925: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
926: PetscObjectGetComm((PetscObject)da,&comm);
928: PetscMalloc1(col*col*nc*nc,&cols);
929: DMGetLocalToGlobalMapping(da,<og);
930: DMGetLocalToGlobalMappingBlock(da,<ogb);
932: /* determine the matrix preallocation information */
933: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
934: for (i=xs; i<xs+nx; i++) {
936: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
937: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
939: for (j=ys; j<ys+ny; j++) {
940: slot = i - gxs + gnx*(j - gys);
942: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
943: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
945: for (k=0; k<nc; k++) {
946: cnt = 0;
947: for (l=lstart; l<lend+1; l++) {
948: for (p=pstart; p<pend+1; p++) {
949: if (l || p) {
950: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
951: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
952: }
953: } else {
954: if (dfill) {
955: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
956: } else {
957: for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
958: }
959: }
960: }
961: }
962: row = k + nc*(slot);
963: MatPreallocateSetLocal(ltog,1,&row,ltog,cnt,cols,dnz,onz);
964: }
965: }
966: }
967: MatSeqAIJSetPreallocation(J,0,dnz);
968: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
969: MatPreallocateFinalize(dnz,onz);
970: MatSetLocalToGlobalMapping(J,ltog,ltog);
971: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
973: /*
974: For each node in the grid: we get the neighbors in the local (on processor ordering
975: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
976: PETSc ordering.
977: */
978: if (!da->prealloc_only) {
979: PetscCalloc1(col*col*nc*nc,&values);
980: for (i=xs; i<xs+nx; i++) {
982: pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
983: pend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
985: for (j=ys; j<ys+ny; j++) {
986: slot = i - gxs + gnx*(j - gys);
988: lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
989: lend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
991: for (k=0; k<nc; k++) {
992: cnt = 0;
993: for (l=lstart; l<lend+1; l++) {
994: for (p=pstart; p<pend+1; p++) {
995: if (l || p) {
996: if ((st == DMDA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
997: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
998: }
999: } else {
1000: if (dfill) {
1001: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
1002: } else {
1003: for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
1004: }
1005: }
1006: }
1007: }
1008: row = k + nc*(slot);
1009: MatSetValuesLocal(J,1,&row,cnt,cols,values,INSERT_VALUES);
1010: }
1011: }
1012: }
1013: PetscFree(values);
1014: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1015: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1016: }
1017: PetscFree(cols);
1018: return(0);
1019: }
1021: /* ---------------------------------------------------------------------------------*/
1025: PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ(DM da,Mat J)
1026: {
1027: PetscErrorCode ierr;
1028: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1029: PetscInt m,n,dim,s,*cols = NULL,k,nc,*rows = NULL,col,cnt,l,p,*dnz = NULL,*onz = NULL;
1030: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1031: MPI_Comm comm;
1032: PetscScalar *values;
1033: DMDABoundaryType bx,by,bz;
1034: ISLocalToGlobalMapping ltog,ltogb;
1035: DMDAStencilType st;
1038: /*
1039: nc - number of components per grid point
1040: col - number of colors needed in one direction for single component problem
1042: */
1043: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1044: col = 2*s + 1;
1046: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1047: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1048: PetscObjectGetComm((PetscObject)da,&comm);
1050: PetscMalloc2(nc,&rows,col*col*col*nc*nc,&cols);
1051: DMGetLocalToGlobalMapping(da,<og);
1052: DMGetLocalToGlobalMappingBlock(da,<ogb);
1054: /* determine the matrix preallocation information */
1055: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1056: for (i=xs; i<xs+nx; i++) {
1057: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1058: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1059: for (j=ys; j<ys+ny; j++) {
1060: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1061: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1062: for (k=zs; k<zs+nz; k++) {
1063: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1064: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1066: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1068: cnt = 0;
1069: for (l=0; l<nc; l++) {
1070: for (ii=istart; ii<iend+1; ii++) {
1071: for (jj=jstart; jj<jend+1; jj++) {
1072: for (kk=kstart; kk<kend+1; kk++) {
1073: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1074: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1075: }
1076: }
1077: }
1078: }
1079: rows[l] = l + nc*(slot);
1080: }
1081: MatPreallocateSetLocal(ltog,nc,rows,ltog,cnt,cols,dnz,onz);
1082: }
1083: }
1084: }
1085: MatSetBlockSize(J,nc);
1086: MatSeqAIJSetPreallocation(J,0,dnz);
1087: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1088: MatPreallocateFinalize(dnz,onz);
1089: MatSetLocalToGlobalMapping(J,ltog,ltog);
1090: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1092: /*
1093: For each node in the grid: we get the neighbors in the local (on processor ordering
1094: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1095: PETSc ordering.
1096: */
1097: if (!da->prealloc_only) {
1098: PetscCalloc1(col*col*col*nc*nc*nc,&values);
1099: for (i=xs; i<xs+nx; i++) {
1100: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1101: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1102: for (j=ys; j<ys+ny; j++) {
1103: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1104: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1105: for (k=zs; k<zs+nz; k++) {
1106: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1107: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1109: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1111: cnt = 0;
1112: for (l=0; l<nc; l++) {
1113: for (ii=istart; ii<iend+1; ii++) {
1114: for (jj=jstart; jj<jend+1; jj++) {
1115: for (kk=kstart; kk<kend+1; kk++) {
1116: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1117: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1118: }
1119: }
1120: }
1121: }
1122: rows[l] = l + nc*(slot);
1123: }
1124: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
1125: }
1126: }
1127: }
1128: PetscFree(values);
1129: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1130: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1131: }
1132: PetscFree2(rows,cols);
1133: return(0);
1134: }
1136: /* ---------------------------------------------------------------------------------*/
1140: PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ_Fill(DM da,Mat J)
1141: {
1142: PetscErrorCode ierr;
1143: DM_DA *dd = (DM_DA*)da->data;
1144: PetscInt xs,nx,i,j,gxs,gnx,row,k,l;
1145: PetscInt m,dim,s,*cols = NULL,nc,col,cnt,*ocols;
1146: PetscInt *ofill = dd->ofill,*dfill = dd->dfill;
1147: PetscScalar *values;
1148: DMDABoundaryType bx;
1149: ISLocalToGlobalMapping ltog,ltogb;
1150: PetscMPIInt rank,size;
1153: if (dd->bx == DMDA_BOUNDARY_PERIODIC) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"With fill provided not implemented with periodic boundary conditions");
1154: MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);
1155: MPI_Comm_size(PetscObjectComm((PetscObject)da),&size);
1157: /*
1158: nc - number of components per grid point
1159: col - number of colors needed in one direction for single component problem
1161: */
1162: DMDAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&bx,0,0,0);
1163: col = 2*s + 1;
1165: DMDAGetCorners(da,&xs,0,0,&nx,0,0);
1166: DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
1168: MatSetBlockSize(J,nc);
1169: PetscCalloc2(nx*nc,&cols,nx*nc,&ocols);
1171: if (nx < 2) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Need at least two grid points per process");
1172: /*
1173: note should be smaller for first and last process with no periodic
1174: does not handle dfill
1175: */
1176: cnt = 0;
1177: /* coupling with process to the left */
1178: for (i=0; i<s; i++) {
1179: for (j=0; j<nc; j++) {
1180: ocols[cnt] = ((!rank) ? 0 : (s - i)*(ofill[j+1] - ofill[j]));
1181: cols[cnt] = dfill[j+1] - dfill[j] + (s + i)*(ofill[j+1] - ofill[j]);
1182: cnt++;
1183: }
1184: }
1185: for (i=s; i<nx-s; i++) {
1186: for (j=0; j<nc; j++) {
1187: cols[cnt] = dfill[j+1] - dfill[j] + 2*s*(ofill[j+1] - ofill[j]);
1188: cnt++;
1189: }
1190: }
1191: /* coupling with process to the right */
1192: for (i=nx-s; i<nx; i++) {
1193: for (j=0; j<nc; j++) {
1194: ocols[cnt] = ((rank == (size-1)) ? 0 : (i - nx + s + 1)*(ofill[j+1] - ofill[j]));
1195: cols[cnt] = dfill[j+1] - dfill[j] + (s + nx - i - 1)*(ofill[j+1] - ofill[j]);
1196: cnt++;
1197: }
1198: }
1200: MatSeqAIJSetPreallocation(J,0,cols);
1201: MatMPIAIJSetPreallocation(J,0,cols,0,ocols);
1202: PetscFree2(cols,ocols);
1204: DMGetLocalToGlobalMapping(da,<og);
1205: DMGetLocalToGlobalMappingBlock(da,<ogb);
1206: MatSetLocalToGlobalMapping(J,ltog,ltog);
1207: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1209: /*
1210: For each node in the grid: we get the neighbors in the local (on processor ordering
1211: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1212: PETSc ordering.
1213: */
1214: if (!da->prealloc_only) {
1215: PetscMalloc1(col*nc*nc,&cols);
1216: PetscCalloc1(col*nc*nc,&values);
1218: row = xs*nc;
1219: /* coupling with process to the left */
1220: for (i=xs; i<xs+s; i++) {
1221: for (j=0; j<nc; j++) {
1222: cnt = 0;
1223: if (rank) {
1224: for (l=0; l<s; l++) {
1225: for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i - s + l)*nc + ofill[k];
1226: }
1227: }
1228: if (dfill) {
1229: for (k=dfill[j]; k<dfill[j+1]; k++) {
1230: cols[cnt++] = i*nc + dfill[k];
1231: }
1232: } else {
1233: for (k=0; k<nc; k++) {
1234: cols[cnt++] = i*nc + k;
1235: }
1236: }
1237: for (l=0; l<s; l++) {
1238: for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i + s - l)*nc + ofill[k];
1239: }
1240: MatSetValues(J,1,&row,cnt,cols,values,INSERT_VALUES);
1241: row++;
1242: }
1243: }
1244: for (i=xs+s; i<xs+nx-s; i++) {
1245: for (j=0; j<nc; j++) {
1246: cnt = 0;
1247: for (l=0; l<s; l++) {
1248: for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i - s + l)*nc + ofill[k];
1249: }
1250: if (dfill) {
1251: for (k=dfill[j]; k<dfill[j+1]; k++) {
1252: cols[cnt++] = i*nc + dfill[k];
1253: }
1254: } else {
1255: for (k=0; k<nc; k++) {
1256: cols[cnt++] = i*nc + k;
1257: }
1258: }
1259: for (l=0; l<s; l++) {
1260: for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i + s - l)*nc + ofill[k];
1261: }
1262: MatSetValues(J,1,&row,cnt,cols,values,INSERT_VALUES);
1263: row++;
1264: }
1265: }
1266: /* coupling with process to the right */
1267: for (i=xs+nx-s; i<xs+nx; i++) {
1268: for (j=0; j<nc; j++) {
1269: cnt = 0;
1270: for (l=0; l<s; l++) {
1271: for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i - s + l)*nc + ofill[k];
1272: }
1273: if (dfill) {
1274: for (k=dfill[j]; k<dfill[j+1]; k++) {
1275: cols[cnt++] = i*nc + dfill[k];
1276: }
1277: } else {
1278: for (k=0; k<nc; k++) {
1279: cols[cnt++] = i*nc + k;
1280: }
1281: }
1282: if (rank < size-1) {
1283: for (l=0; l<s; l++) {
1284: for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i + s - l)*nc + ofill[k];
1285: }
1286: }
1287: MatSetValues(J,1,&row,cnt,cols,values,INSERT_VALUES);
1288: row++;
1289: }
1290: }
1291: PetscFree(values);
1292: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1293: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1294: PetscFree(cols);
1295: }
1296: return(0);
1297: }
1299: /* ---------------------------------------------------------------------------------*/
1303: PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ(DM da,Mat J)
1304: {
1305: PetscErrorCode ierr;
1306: PetscInt xs,nx,i,i1,slot,gxs,gnx;
1307: PetscInt m,dim,s,*cols = NULL,nc,*rows = NULL,col,cnt,l;
1308: PetscInt istart,iend;
1309: PetscScalar *values;
1310: DMDABoundaryType bx;
1311: ISLocalToGlobalMapping ltog,ltogb;
1314: /*
1315: nc - number of components per grid point
1316: col - number of colors needed in one direction for single component problem
1318: */
1319: DMDAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&bx,0,0,0);
1320: col = 2*s + 1;
1322: DMDAGetCorners(da,&xs,0,0,&nx,0,0);
1323: DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
1325: MatSetBlockSize(J,nc);
1326: MatSeqAIJSetPreallocation(J,col*nc,0);
1327: MatMPIAIJSetPreallocation(J,col*nc,0,col*nc,0);
1329: DMGetLocalToGlobalMapping(da,<og);
1330: DMGetLocalToGlobalMappingBlock(da,<ogb);
1331: MatSetLocalToGlobalMapping(J,ltog,ltog);
1332: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1334: /*
1335: For each node in the grid: we get the neighbors in the local (on processor ordering
1336: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1337: PETSc ordering.
1338: */
1339: if (!da->prealloc_only) {
1340: PetscMalloc2(nc,&rows,col*nc*nc,&cols);
1341: PetscCalloc1(col*nc*nc,&values);
1342: for (i=xs; i<xs+nx; i++) {
1343: istart = PetscMax(-s,gxs - i);
1344: iend = PetscMin(s,gxs + gnx - i - 1);
1345: slot = i - gxs;
1347: cnt = 0;
1348: for (l=0; l<nc; l++) {
1349: for (i1=istart; i1<iend+1; i1++) {
1350: cols[cnt++] = l + nc*(slot + i1);
1351: }
1352: rows[l] = l + nc*(slot);
1353: }
1354: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
1355: }
1356: PetscFree(values);
1357: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1358: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1359: PetscFree2(rows,cols);
1360: }
1361: return(0);
1362: }
1366: PetscErrorCode DMCreateMatrix_DA_2d_MPIBAIJ(DM da,Mat J)
1367: {
1368: PetscErrorCode ierr;
1369: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1370: PetscInt m,n,dim,s,*cols,nc,col,cnt,*dnz,*onz;
1371: PetscInt istart,iend,jstart,jend,ii,jj;
1372: MPI_Comm comm;
1373: PetscScalar *values;
1374: DMDABoundaryType bx,by;
1375: DMDAStencilType st;
1376: ISLocalToGlobalMapping ltog,ltogb;
1379: /*
1380: nc - number of components per grid point
1381: col - number of colors needed in one direction for single component problem
1382: */
1383: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
1384: col = 2*s + 1;
1386: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
1387: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
1388: PetscObjectGetComm((PetscObject)da,&comm);
1390: PetscMalloc1(col*col*nc*nc,&cols);
1392: DMGetLocalToGlobalMapping(da,<og);
1393: DMGetLocalToGlobalMappingBlock(da,<ogb);
1395: /* determine the matrix preallocation information */
1396: MatPreallocateInitialize(comm,nx*ny,nx*ny,dnz,onz);
1397: for (i=xs; i<xs+nx; i++) {
1398: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1399: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1400: for (j=ys; j<ys+ny; j++) {
1401: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1402: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1403: slot = i - gxs + gnx*(j - gys);
1405: /* Find block columns in block row */
1406: cnt = 0;
1407: for (ii=istart; ii<iend+1; ii++) {
1408: for (jj=jstart; jj<jend+1; jj++) {
1409: if (st == DMDA_STENCIL_BOX || !ii || !jj) { /* BOX or on the STAR */
1410: cols[cnt++] = slot + ii + gnx*jj;
1411: }
1412: }
1413: }
1414: MatPreallocateSetLocal(ltogb,1,&slot,ltogb,cnt,cols,dnz,onz);
1415: }
1416: }
1417: MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1418: MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1419: MatPreallocateFinalize(dnz,onz);
1421: MatSetLocalToGlobalMapping(J,ltog,ltog);
1422: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1424: /*
1425: For each node in the grid: we get the neighbors in the local (on processor ordering
1426: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1427: PETSc ordering.
1428: */
1429: if (!da->prealloc_only) {
1430: PetscCalloc1(col*col*nc*nc,&values);
1431: for (i=xs; i<xs+nx; i++) {
1432: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1433: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1434: for (j=ys; j<ys+ny; j++) {
1435: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1436: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1437: slot = i - gxs + gnx*(j - gys);
1438: cnt = 0;
1439: for (ii=istart; ii<iend+1; ii++) {
1440: for (jj=jstart; jj<jend+1; jj++) {
1441: if (st == DMDA_STENCIL_BOX || !ii || !jj) { /* BOX or on the STAR */
1442: cols[cnt++] = slot + ii + gnx*jj;
1443: }
1444: }
1445: }
1446: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1447: }
1448: }
1449: PetscFree(values);
1450: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1451: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1452: }
1453: PetscFree(cols);
1454: return(0);
1455: }
1459: PetscErrorCode DMCreateMatrix_DA_3d_MPIBAIJ(DM da,Mat J)
1460: {
1461: PetscErrorCode ierr;
1462: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1463: PetscInt m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1464: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1465: MPI_Comm comm;
1466: PetscScalar *values;
1467: DMDABoundaryType bx,by,bz;
1468: DMDAStencilType st;
1469: ISLocalToGlobalMapping ltog,ltogb;
1472: /*
1473: nc - number of components per grid point
1474: col - number of colors needed in one direction for single component problem
1476: */
1477: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1478: col = 2*s + 1;
1480: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1481: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1482: PetscObjectGetComm((PetscObject)da,&comm);
1484: PetscMalloc1(col*col*col,&cols);
1486: DMGetLocalToGlobalMapping(da,<og);
1487: DMGetLocalToGlobalMappingBlock(da,<ogb);
1489: /* determine the matrix preallocation information */
1490: MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1491: for (i=xs; i<xs+nx; i++) {
1492: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1493: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1494: for (j=ys; j<ys+ny; j++) {
1495: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1496: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1497: for (k=zs; k<zs+nz; k++) {
1498: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1499: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1501: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1503: /* Find block columns in block row */
1504: cnt = 0;
1505: for (ii=istart; ii<iend+1; ii++) {
1506: for (jj=jstart; jj<jend+1; jj++) {
1507: for (kk=kstart; kk<kend+1; kk++) {
1508: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1509: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1510: }
1511: }
1512: }
1513: }
1514: MatPreallocateSetLocal(ltogb,1,&slot,ltogb,cnt,cols,dnz,onz);
1515: }
1516: }
1517: }
1518: MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1519: MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1520: MatPreallocateFinalize(dnz,onz);
1522: MatSetLocalToGlobalMapping(J,ltog,ltog);
1523: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1525: /*
1526: For each node in the grid: we get the neighbors in the local (on processor ordering
1527: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1528: PETSc ordering.
1529: */
1530: if (!da->prealloc_only) {
1531: PetscCalloc1(col*col*col*nc*nc,&values);
1532: for (i=xs; i<xs+nx; i++) {
1533: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1534: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1535: for (j=ys; j<ys+ny; j++) {
1536: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1537: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1538: for (k=zs; k<zs+nz; k++) {
1539: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1540: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1542: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1544: cnt = 0;
1545: for (ii=istart; ii<iend+1; ii++) {
1546: for (jj=jstart; jj<jend+1; jj++) {
1547: for (kk=kstart; kk<kend+1; kk++) {
1548: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1549: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1550: }
1551: }
1552: }
1553: }
1554: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1555: }
1556: }
1557: }
1558: PetscFree(values);
1559: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1560: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1561: }
1562: PetscFree(cols);
1563: return(0);
1564: }
1568: /*
1569: This helper is for of SBAIJ preallocation, to discard the lower-triangular values which are difficult to
1570: identify in the local ordering with periodic domain.
1571: */
1572: static PetscErrorCode L2GFilterUpperTriangular(ISLocalToGlobalMapping ltog,PetscInt *row,PetscInt *cnt,PetscInt col[])
1573: {
1575: PetscInt i,n;
1578: ISLocalToGlobalMappingApply(ltog,1,row,row);
1579: ISLocalToGlobalMappingApply(ltog,*cnt,col,col);
1580: for (i=0,n=0; i<*cnt; i++) {
1581: if (col[i] >= *row) col[n++] = col[i];
1582: }
1583: *cnt = n;
1584: return(0);
1585: }
1589: PetscErrorCode DMCreateMatrix_DA_2d_MPISBAIJ(DM da,Mat J)
1590: {
1591: PetscErrorCode ierr;
1592: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1593: PetscInt m,n,dim,s,*cols,nc,col,cnt,*dnz,*onz;
1594: PetscInt istart,iend,jstart,jend,ii,jj;
1595: MPI_Comm comm;
1596: PetscScalar *values;
1597: DMDABoundaryType bx,by;
1598: DMDAStencilType st;
1599: ISLocalToGlobalMapping ltog,ltogb;
1602: /*
1603: nc - number of components per grid point
1604: col - number of colors needed in one direction for single component problem
1605: */
1606: DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
1607: col = 2*s + 1;
1609: DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
1610: DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
1611: PetscObjectGetComm((PetscObject)da,&comm);
1613: PetscMalloc1(col*col*nc*nc,&cols);
1615: DMGetLocalToGlobalMapping(da,<og);
1616: DMGetLocalToGlobalMappingBlock(da,<ogb);
1618: /* determine the matrix preallocation information */
1619: MatPreallocateInitialize(comm,nx*ny,nx*ny,dnz,onz);
1620: for (i=xs; i<xs+nx; i++) {
1621: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1622: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1623: for (j=ys; j<ys+ny; j++) {
1624: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1625: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1626: slot = i - gxs + gnx*(j - gys);
1628: /* Find block columns in block row */
1629: cnt = 0;
1630: for (ii=istart; ii<iend+1; ii++) {
1631: for (jj=jstart; jj<jend+1; jj++) {
1632: if (st == DMDA_STENCIL_BOX || !ii || !jj) {
1633: cols[cnt++] = slot + ii + gnx*jj;
1634: }
1635: }
1636: }
1637: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1638: MatPreallocateSymmetricSet(slot,cnt,cols,dnz,onz);
1639: }
1640: }
1641: MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1642: MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1643: MatPreallocateFinalize(dnz,onz);
1645: MatSetLocalToGlobalMapping(J,ltog,ltog);
1646: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1648: /*
1649: For each node in the grid: we get the neighbors in the local (on processor ordering
1650: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1651: PETSc ordering.
1652: */
1653: if (!da->prealloc_only) {
1654: PetscCalloc1(col*col*nc*nc,&values);
1655: for (i=xs; i<xs+nx; i++) {
1656: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1657: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1658: for (j=ys; j<ys+ny; j++) {
1659: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1660: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1661: slot = i - gxs + gnx*(j - gys);
1663: /* Find block columns in block row */
1664: cnt = 0;
1665: for (ii=istart; ii<iend+1; ii++) {
1666: for (jj=jstart; jj<jend+1; jj++) {
1667: if (st == DMDA_STENCIL_BOX || !ii || !jj) {
1668: cols[cnt++] = slot + ii + gnx*jj;
1669: }
1670: }
1671: }
1672: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1673: MatSetValuesBlocked(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1674: }
1675: }
1676: PetscFree(values);
1677: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1678: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1679: }
1680: PetscFree(cols);
1681: return(0);
1682: }
1686: PetscErrorCode DMCreateMatrix_DA_3d_MPISBAIJ(DM da,Mat J)
1687: {
1688: PetscErrorCode ierr;
1689: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1690: PetscInt m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1691: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1692: MPI_Comm comm;
1693: PetscScalar *values;
1694: DMDABoundaryType bx,by,bz;
1695: DMDAStencilType st;
1696: ISLocalToGlobalMapping ltog,ltogb;
1699: /*
1700: nc - number of components per grid point
1701: col - number of colors needed in one direction for single component problem
1702: */
1703: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1704: col = 2*s + 1;
1706: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1707: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1708: PetscObjectGetComm((PetscObject)da,&comm);
1710: /* create the matrix */
1711: PetscMalloc1(col*col*col,&cols);
1713: DMGetLocalToGlobalMapping(da,<og);
1714: DMGetLocalToGlobalMappingBlock(da,<ogb);
1716: /* determine the matrix preallocation information */
1717: MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1718: for (i=xs; i<xs+nx; i++) {
1719: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1720: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1721: for (j=ys; j<ys+ny; j++) {
1722: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1723: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1724: for (k=zs; k<zs+nz; k++) {
1725: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1726: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1728: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1730: /* Find block columns in block row */
1731: cnt = 0;
1732: for (ii=istart; ii<iend+1; ii++) {
1733: for (jj=jstart; jj<jend+1; jj++) {
1734: for (kk=kstart; kk<kend+1; kk++) {
1735: if ((st == DMDA_STENCIL_BOX) || (!ii && !jj) || (!jj && !kk) || (!ii && !kk)) {
1736: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1737: }
1738: }
1739: }
1740: }
1741: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1742: MatPreallocateSymmetricSet(slot,cnt,cols,dnz,onz);
1743: }
1744: }
1745: }
1746: MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1747: MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1748: MatPreallocateFinalize(dnz,onz);
1750: MatSetLocalToGlobalMapping(J,ltog,ltog);
1751: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1753: /*
1754: For each node in the grid: we get the neighbors in the local (on processor ordering
1755: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1756: PETSc ordering.
1757: */
1758: if (!da->prealloc_only) {
1759: PetscCalloc1(col*col*col*nc*nc,&values);
1760: for (i=xs; i<xs+nx; i++) {
1761: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1762: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1763: for (j=ys; j<ys+ny; j++) {
1764: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1765: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1766: for (k=zs; k<zs+nz; k++) {
1767: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1768: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1770: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1772: cnt = 0;
1773: for (ii=istart; ii<iend+1; ii++) {
1774: for (jj=jstart; jj<jend+1; jj++) {
1775: for (kk=kstart; kk<kend+1; kk++) {
1776: if ((st == DMDA_STENCIL_BOX) || (!ii && !jj) || (!jj && !kk) || (!ii && !kk)) {
1777: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1778: }
1779: }
1780: }
1781: }
1782: L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1783: MatSetValuesBlocked(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1784: }
1785: }
1786: }
1787: PetscFree(values);
1788: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1789: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1790: }
1791: PetscFree(cols);
1792: return(0);
1793: }
1795: /* ---------------------------------------------------------------------------------*/
1799: PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ_Fill(DM da,Mat J)
1800: {
1801: PetscErrorCode ierr;
1802: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1803: PetscInt m,n,dim,s,*cols,k,nc,row,col,cnt,l,p,*dnz,*onz;
1804: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1805: DM_DA *dd = (DM_DA*)da->data;
1806: PetscInt ifill_col,*dfill = dd->dfill,*ofill = dd->ofill;
1807: MPI_Comm comm;
1808: PetscScalar *values;
1809: DMDABoundaryType bx,by,bz;
1810: ISLocalToGlobalMapping ltog,ltogb;
1811: DMDAStencilType st;
1814: /*
1815: nc - number of components per grid point
1816: col - number of colors needed in one direction for single component problem
1818: */
1819: DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1820: col = 2*s + 1;
1821: if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1822: by 2*stencil_width + 1\n");
1823: if (by == DMDA_BOUNDARY_PERIODIC && (n % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1824: by 2*stencil_width + 1\n");
1825: if (bz == DMDA_BOUNDARY_PERIODIC && (p % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1826: by 2*stencil_width + 1\n");
1828: DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1829: DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1830: PetscObjectGetComm((PetscObject)da,&comm);
1832: PetscMalloc1(col*col*col*nc,&cols);
1833: DMGetLocalToGlobalMapping(da,<og);
1834: DMGetLocalToGlobalMappingBlock(da,<ogb);
1836: /* determine the matrix preallocation information */
1837: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1840: for (i=xs; i<xs+nx; i++) {
1841: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1842: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1843: for (j=ys; j<ys+ny; j++) {
1844: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1845: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1846: for (k=zs; k<zs+nz; k++) {
1847: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1848: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1850: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1852: for (l=0; l<nc; l++) {
1853: cnt = 0;
1854: for (ii=istart; ii<iend+1; ii++) {
1855: for (jj=jstart; jj<jend+1; jj++) {
1856: for (kk=kstart; kk<kend+1; kk++) {
1857: if (ii || jj || kk) {
1858: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1859: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1860: }
1861: } else {
1862: if (dfill) {
1863: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1864: } else {
1865: for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1866: }
1867: }
1868: }
1869: }
1870: }
1871: row = l + nc*(slot);
1872: MatPreallocateSetLocal(ltog,1,&row,ltog,cnt,cols,dnz,onz);
1873: }
1874: }
1875: }
1876: }
1877: MatSeqAIJSetPreallocation(J,0,dnz);
1878: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1879: MatPreallocateFinalize(dnz,onz);
1880: MatSetLocalToGlobalMapping(J,ltog,ltog);
1881: MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);
1883: /*
1884: For each node in the grid: we get the neighbors in the local (on processor ordering
1885: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1886: PETSc ordering.
1887: */
1888: if (!da->prealloc_only) {
1889: PetscCalloc1(col*col*col*nc*nc*nc,&values);
1890: for (i=xs; i<xs+nx; i++) {
1891: istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1892: iend = (bx == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,m-i-1));
1893: for (j=ys; j<ys+ny; j++) {
1894: jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1895: jend = (by == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,n-j-1));
1896: for (k=zs; k<zs+nz; k++) {
1897: kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1898: kend = (bz == DMDA_BOUNDARY_PERIODIC) ? s : (PetscMin(s,p-k-1));
1900: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1902: for (l=0; l<nc; l++) {
1903: cnt = 0;
1904: for (ii=istart; ii<iend+1; ii++) {
1905: for (jj=jstart; jj<jend+1; jj++) {
1906: for (kk=kstart; kk<kend+1; kk++) {
1907: if (ii || jj || kk) {
1908: if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1909: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1910: }
1911: } else {
1912: if (dfill) {
1913: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1914: } else {
1915: for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1916: }
1917: }
1918: }
1919: }
1920: }
1921: row = l + nc*(slot);
1922: MatSetValuesLocal(J,1,&row,cnt,cols,values,INSERT_VALUES);
1923: }
1924: }
1925: }
1926: }
1927: PetscFree(values);
1928: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1929: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1930: }
1931: PetscFree(cols);
1932: return(0);
1933: }