Actual source code: fdda.c
1:
2: #include src/dm/da/daimpl.h
3: #include petscmat.h
6: EXTERN PetscErrorCode DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
7: EXTERN PetscErrorCode DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
8: EXTERN PetscErrorCode DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
9: EXTERN PetscErrorCode DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);
11: /*
12: For ghost i that may be negative or greater than the upper bound this
13: maps it into the 0:m-1 range using periodicity
14: */
15: #define SetInRange(i,m) ((i < 0) ? m+i:((i >= m) ? i-m:i))
19: static PetscErrorCode DASetBlockFills_Private(PetscInt *dfill,PetscInt w,PetscInt **rfill)
20: {
22: PetscInt i,j,nz,*fill;
25: /* count number nonzeros */
26: nz = 0;
27: for (i=0; i<w; i++) {
28: for (j=0; j<w; j++) {
29: if (dfill[w*i+j]) nz++;
30: }
31: }
32: PetscMalloc((nz + w + 1)*sizeof(PetscInt),&fill);
33: /* construct modified CSR storage of nonzero structure */
34: nz = w + 1;
35: for (i=0; i<w; i++) {
36: fill[i] = nz;
37: for (j=0; j<w; j++) {
38: if (dfill[w*i+j]) {
39: fill[nz] = j;
40: nz++;
41: }
42: }
43: }
44: fill[w] = nz;
45:
46: *rfill = fill;
47: return(0);
48: }
52: /*@C
53: DASetBlockFills - Sets the fill pattern in each block for a multi-component problem
54: of the matrix returned by DAGetMatrix().
56: Collective on DA
58: Input Parameter:
59: + da - the distributed array
60: . dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
61: - ofill - the fill pattern in the off-diagonal blocks
64: Level: developer
66: Notes: This only makes sense when you are doing multicomponent problems but using the
67: MPIAIJ matrix format
69: The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
70: representing coupling and 0 entries for missing coupling. For example
71: $ dfill[3][3] = {1, 0, 0,
72: $ 1, 1, 0,
73: $ 0, 1, 1}
74: means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with
75: itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the
76: diagonal block.
78: DASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
79: can be represented in the dfill, ofill format
81: Contributed by Glenn Hammond
83: .seealso DAGetMatrix(), DASetGetMatrix()
85: @*/
86: PetscErrorCode DASetBlockFills(DA da,PetscInt *dfill,PetscInt *ofill)
87: {
91: if (dfill) {
92: DASetBlockFills_Private(dfill,da->w,&da->dfill);
93: }
94: DASetBlockFills_Private(ofill,da->w,&da->ofill);
95: return(0);
96: }
101: /*@C
102: DAGetColoring - Gets the coloring required for computing the Jacobian via
103: finite differences on a function defined using a stencil on the DA.
105: Collective on DA
107: Input Parameter:
108: + da - the distributed array
109: - ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED
111: Output Parameters:
112: . coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)
114: Level: advanced
116: Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used
117: for efficient (parallel or thread based) triangular solves etc is NOT yet
118: available.
121: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring
123: @*/
124: PetscErrorCode DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
125: {
127: PetscInt dim;
130: /*
131: m
132: ------------------------------------------------------
133: | |
134: | |
135: | ---------------------- |
136: | | | |
137: n | yn | | |
138: | | | |
139: | .--------------------- |
140: | (xs,ys) xn |
141: | . |
142: | (gxs,gys) |
143: | |
144: -----------------------------------------------------
145: */
147: /*
148: nc - number of components per grid point
149: col - number of colors needed in one direction for single component problem
150:
151: */
152: DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);
154: /*
155: We do not provide a getcoloring function in the DA operations because
156: the basic DA does not know about matrices. We think of DA as being more
157: more low-level then matrices.
158: */
159: if (dim == 1) {
160: DAGetColoring1d_MPIAIJ(da,ctype,coloring);
161: } else if (dim == 2) {
162: DAGetColoring2d_MPIAIJ(da,ctype,coloring);
163: } else if (dim == 3) {
164: DAGetColoring3d_MPIAIJ(da,ctype,coloring);
165: } else {
166: SETERRQ1(PETSC_ERR_SUP,"Not done for %D dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
167: }
168: return(0);
169: }
171: /* ---------------------------------------------------------------------------------*/
175: PetscErrorCode DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
176: {
177: PetscErrorCode ierr;
178: PetscInt xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,M,N,dim,s,k,nc,col;
179: PetscMPIInt size;
180: MPI_Comm comm;
181: DAPeriodicType wrap;
182: DAStencilType st;
183: ISColoringValue *colors;
186: /*
187: nc - number of components per grid point
188: col - number of colors needed in one direction for single component problem
189:
190: */
191: DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&wrap,&st);
192: col = 2*s + 1;
193: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
194: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
195: PetscObjectGetComm((PetscObject)da,&comm);
196: MPI_Comm_size(comm,&size);
198: /* special case as taught to us by Paul Hovland */
199: if (st == DA_STENCIL_STAR && s == 1) {
200: DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
201: } else {
203: if (DAXPeriodic(wrap) && (m % col)){
204: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
205: by 2*stencil_width + 1\n");
206: }
207: if (DAYPeriodic(wrap) && (n % col)){
208: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
209: by 2*stencil_width + 1\n");
210: }
211: if (ctype == IS_COLORING_LOCAL) {
212: if (!da->localcoloring) {
213: PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
214: ii = 0;
215: for (j=ys; j<ys+ny; j++) {
216: for (i=xs; i<xs+nx; i++) {
217: for (k=0; k<nc; k++) {
218: colors[ii++] = k + nc*((i % col) + col*(j % col));
219: }
220: }
221: }
222: ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
223: }
224: *coloring = da->localcoloring;
225: } else if (ctype == IS_COLORING_GHOSTED) {
226: if (!da->ghostedcoloring) {
227: PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
228: ii = 0;
229: for (j=gys; j<gys+gny; j++) {
230: for (i=gxs; i<gxs+gnx; i++) {
231: for (k=0; k<nc; k++) {
232: /* the complicated stuff is to handle periodic boundaries */
233: colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
234: }
235: }
236: }
237: ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
238: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
239: }
240: *coloring = da->ghostedcoloring;
241: } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
242: }
243: ISColoringReference(*coloring);
244: return(0);
245: }
247: /* ---------------------------------------------------------------------------------*/
251: PetscErrorCode DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
252: {
253: PetscErrorCode ierr;
254: PetscInt xs,ys,nx,ny,i,j,gxs,gys,gnx,gny,m,n,p,dim,s,k,nc,col,zs,gzs,ii,l,nz,gnz,M,N,P;
255: PetscMPIInt size;
256: MPI_Comm comm;
257: DAPeriodicType wrap;
258: DAStencilType st;
259: ISColoringValue *colors;
262: /*
263: nc - number of components per grid point
264: col - number of colors needed in one direction for single component problem
265:
266: */
267: DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
268: col = 2*s + 1;
269: if (DAXPeriodic(wrap) && (m % col)){
270: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
271: by 2*stencil_width + 1\n");
272: }
273: if (DAYPeriodic(wrap) && (n % col)){
274: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
275: by 2*stencil_width + 1\n");
276: }
277: if (DAZPeriodic(wrap) && (p % col)){
278: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
279: by 2*stencil_width + 1\n");
280: }
282: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
283: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
284: PetscObjectGetComm((PetscObject)da,&comm);
285: MPI_Comm_size(comm,&size);
287: /* create the coloring */
288: if (ctype == IS_COLORING_LOCAL) {
289: if (!da->localcoloring) {
290: PetscMalloc(nc*nx*ny*nz*sizeof(ISColoringValue),&colors);
291: ii = 0;
292: for (k=zs; k<zs+nz; k++) {
293: for (j=ys; j<ys+ny; j++) {
294: for (i=xs; i<xs+nx; i++) {
295: for (l=0; l<nc; l++) {
296: colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
297: }
298: }
299: }
300: }
301: ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
302: }
303: *coloring = da->localcoloring;
304: } else if (ctype == IS_COLORING_GHOSTED) {
305: if (!da->ghostedcoloring) {
306: PetscMalloc(nc*gnx*gny*gnz*sizeof(ISColoringValue),&colors);
307: ii = 0;
308: for (k=gzs; k<gzs+gnz; k++) {
309: for (j=gys; j<gys+gny; j++) {
310: for (i=gxs; i<gxs+gnx; i++) {
311: for (l=0; l<nc; l++) {
312: /* the complicated stuff is to handle periodic boundaries */
313: colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
314: }
315: }
316: }
317: }
318: ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
319: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
320: }
321: *coloring = da->ghostedcoloring;
322: } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
323: ISColoringReference(*coloring);
324: return(0);
325: }
327: /* ---------------------------------------------------------------------------------*/
331: PetscErrorCode DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
332: {
333: PetscErrorCode ierr;
334: PetscInt xs,nx,i,i1,gxs,gnx,l,m,M,dim,s,nc,col;
335: PetscMPIInt size;
336: MPI_Comm comm;
337: DAPeriodicType wrap;
338: ISColoringValue *colors;
341: /*
342: nc - number of components per grid point
343: col - number of colors needed in one direction for single component problem
344:
345: */
346: DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
347: col = 2*s + 1;
349: if (DAXPeriodic(wrap) && (m % col)) {
350: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisible\n\
351: by 2*stencil_width + 1\n");
352: }
354: DAGetCorners(da,&xs,0,0,&nx,0,0);
355: DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
356: PetscObjectGetComm((PetscObject)da,&comm);
357: MPI_Comm_size(comm,&size);
359: /* create the coloring */
360: if (ctype == IS_COLORING_LOCAL) {
361: if (!da->localcoloring) {
362: PetscMalloc(nc*nx*sizeof(ISColoringValue),&colors);
363: i1 = 0;
364: for (i=xs; i<xs+nx; i++) {
365: for (l=0; l<nc; l++) {
366: colors[i1++] = l + nc*(i % col);
367: }
368: }
369: ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
370: }
371: *coloring = da->localcoloring;
372: } else if (ctype == IS_COLORING_GHOSTED) {
373: if (!da->ghostedcoloring) {
374: PetscMalloc(nc*gnx*sizeof(ISColoringValue),&colors);
375: i1 = 0;
376: for (i=gxs; i<gxs+gnx; i++) {
377: for (l=0; l<nc; l++) {
378: /* the complicated stuff is to handle periodic boundaries */
379: colors[i1++] = l + nc*(SetInRange(i,m) % col);
380: }
381: }
382: ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
383: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
384: }
385: *coloring = da->ghostedcoloring;
386: } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
387: ISColoringReference(*coloring);
388: return(0);
389: }
393: PetscErrorCode DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
394: {
395: PetscErrorCode ierr;
396: PetscInt xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,dim,s,k,nc;
397: MPI_Comm comm;
398: DAPeriodicType wrap;
399: ISColoringValue *colors;
402: /*
403: nc - number of components per grid point
404: col - number of colors needed in one direction for single component problem
405:
406: */
407: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,0);
408: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
409: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
410: PetscObjectGetComm((PetscObject)da,&comm);
412: if (DAXPeriodic(wrap) && (m % 5)){
413: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
414: by 5\n");
415: }
416: if (DAYPeriodic(wrap) && (n % 5)){
417: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
418: by 5\n");
419: }
421: /* create the coloring */
422: if (ctype == IS_COLORING_LOCAL) {
423: if (!da->localcoloring) {
424: PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
425: ii = 0;
426: for (j=ys; j<ys+ny; j++) {
427: for (i=xs; i<xs+nx; i++) {
428: for (k=0; k<nc; k++) {
429: colors[ii++] = k + nc*((3*j+i) % 5);
430: }
431: }
432: }
433: ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
434: }
435: *coloring = da->localcoloring;
436: } else if (ctype == IS_COLORING_GHOSTED) {
437: if (!da->ghostedcoloring) {
438: PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
439: ii = 0;
440: for (j=gys; j<gys+gny; j++) {
441: for (i=gxs; i<gxs+gnx; i++) {
442: for (k=0; k<nc; k++) {
443: colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
444: }
445: }
446: }
447: ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
448: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
449: }
450: *coloring = da->ghostedcoloring;
451: } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
452: return(0);
453: }
455: /* =========================================================================== */
456: EXTERN PetscErrorCode DAGetMatrix1d_MPIAIJ(DA,Mat);
457: EXTERN PetscErrorCode DAGetMatrix2d_MPIAIJ(DA,Mat);
458: EXTERN PetscErrorCode DAGetMatrix2d_MPIAIJ_Fill(DA,Mat);
459: EXTERN PetscErrorCode DAGetMatrix3d_MPIAIJ(DA,Mat);
460: EXTERN PetscErrorCode DAGetMatrix3d_MPIAIJ_Fill(DA,Mat);
461: EXTERN PetscErrorCode DAGetMatrix3d_MPIBAIJ(DA,Mat);
462: EXTERN PetscErrorCode DAGetMatrix3d_MPISBAIJ(DA,Mat);
466: /*@C
467: DAGetMatrix - Creates a matrix with the correct parallel layout and nonzero structure required for
468: computing the Jacobian on a function defined using the stencil set in the DA.
470: Collective on DA
472: Input Parameter:
473: + da - the distributed array
474: - mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, or
475: any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).
477: Output Parameters:
478: . J - matrix with the correct nonzero structure
479: (obviously without the correct Jacobian values)
481: Level: advanced
483: Notes: This properly preallocates the number of nonzeros in the sparse matrix so you
484: do not need to do it yourself.
486: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()
488: @*/
489: PetscErrorCode DAGetMatrix(DA da,const MatType mtype,Mat *J)
490: {
492: PetscInt dim,dof,nx,ny,nz,dims[3],starts[3];
493: Mat A;
494: MPI_Comm comm;
495: MatType Atype;
496: void (*aij)(void)=PETSC_NULL,(*baij)(void)=PETSC_NULL,(*sbaij)(void)=PETSC_NULL;
499: /*
500: m
501: ------------------------------------------------------
502: | |
503: | |
504: | ---------------------- |
505: | | | |
506: n | yn | | |
507: | | | |
508: | .--------------------- |
509: | (xs,ys) xn |
510: | . |
511: | (gxs,gys) |
512: | |
513: -----------------------------------------------------
514: */
516: /*
517: nc - number of components per grid point
518: col - number of colors needed in one direction for single component problem
519:
520: */
521: DAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0);
522: DAGetCorners(da,0,0,0,&nx,&ny,&nz);
523: PetscObjectGetComm((PetscObject)da,&comm);
524: MatCreate(comm,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,&A);
525: MatSetType(A,mtype);
526: MatSetFromOptions(A);
527: MatGetType(A,&Atype);
528: /*
529: We do not provide a getmatrix function in the DA operations because
530: the basic DA does not know about matrices. We think of DA as being more
531: more low-level then matrices. This is kind of cheating but, cause sometimes
532: we think of DA has higher level then matrices.
534: We could switch based on Atype (or mtype), but we do not since the
535: specialized setting routines depend only the particular preallocation
536: details of the matrix, not the type itself.
537: */
538: PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
539: if (!aij) {
540: PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
541: }
542: if (aij) {
543: if (dim == 1) {
544: DAGetMatrix1d_MPIAIJ(da,A);
545: } else if (dim == 2) {
546: if (da->ofill) {
547: DAGetMatrix2d_MPIAIJ_Fill(da,A);
548: } else {
549: DAGetMatrix2d_MPIAIJ(da,A);
550: }
551: } else if (dim == 3) {
552: if (da->ofill) {
553: DAGetMatrix3d_MPIAIJ_Fill(da,A);
554: } else {
555: DAGetMatrix3d_MPIAIJ(da,A);
556: }
557: }
558: } else if (dim == 3) {
559: PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
560: if (!baij) {
561: PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
562: }
563: if (baij) {
564: DAGetMatrix3d_MPIBAIJ(da,A);
565: } else {
566: PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
567: if (!sbaij) {
568: PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
569: }
570: if (sbaij) {
571: DAGetMatrix3d_MPISBAIJ(da,A);
572: } else {
573: SETERRQ2(PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s!\n" \
574: "Send mail to petsc-maint@mcs.anl.gov for code",
575: dim,Atype);
576: }
577: }
578: } else {
579: SETERRQ2(PETSC_ERR_SUP,"Not implemented for %D dimension and matrix type: %s!\n" \
580: "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype);
581: }
582: DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
583: MatSetStencil(A,dim,dims,starts,dof);
584: *J = A;
585: return(0);
586: }
588: /* ---------------------------------------------------------------------------------*/
591: PetscErrorCode DAGetMatrix2d_MPIAIJ(DA da,Mat J)
592: {
593: PetscErrorCode ierr;
594: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny,m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
595: PetscInt lstart,lend,pstart,pend,*dnz,*onz;
596: MPI_Comm comm;
597: PetscScalar *values;
598: DAPeriodicType wrap;
599: ISLocalToGlobalMapping ltog;
600: DAStencilType st;
603: /*
604: nc - number of components per grid point
605: col - number of colors needed in one direction for single component problem
606:
607: */
608: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
609: col = 2*s + 1;
610: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
611: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
612: PetscObjectGetComm((PetscObject)da,&comm);
614: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
615: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
616: PetscMalloc(nc*sizeof(PetscInt),&rows);
617: PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
618: DAGetISLocalToGlobalMapping(da,<og);
619:
620: /* determine the matrix preallocation information */
621: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
622: for (i=xs; i<xs+nx; i++) {
624: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
625: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
627: for (j=ys; j<ys+ny; j++) {
628: slot = i - gxs + gnx*(j - gys);
630: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
631: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
633: cnt = 0;
634: for (k=0; k<nc; k++) {
635: for (l=lstart; l<lend+1; l++) {
636: for (p=pstart; p<pend+1; p++) {
637: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
638: cols[cnt++] = k + nc*(slot + gnx*l + p);
639: }
640: }
641: }
642: rows[k] = k + nc*(slot);
643: }
644: MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
645: }
646: }
647: MatSeqAIJSetPreallocation(J,0,dnz);
648: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
649: MatSetBlockSize(J,nc);
650: MatPreallocateFinalize(dnz,onz);
652: MatSetLocalToGlobalMapping(J,ltog);
654: /*
655: For each node in the grid: we get the neighbors in the local (on processor ordering
656: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
657: PETSc ordering.
658: */
659: for (i=xs; i<xs+nx; i++) {
660:
661: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
662: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
663:
664: for (j=ys; j<ys+ny; j++) {
665: slot = i - gxs + gnx*(j - gys);
666:
667: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
668: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
670: cnt = 0;
671: for (k=0; k<nc; k++) {
672: for (l=lstart; l<lend+1; l++) {
673: for (p=pstart; p<pend+1; p++) {
674: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
675: cols[cnt++] = k + nc*(slot + gnx*l + p);
676: }
677: }
678: }
679: rows[k] = k + nc*(slot);
680: }
681: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
682: }
683: }
684: PetscFree(values);
685: PetscFree(rows);
686: PetscFree(cols);
687: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
688: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
689: return(0);
690: }
694: PetscErrorCode DAGetMatrix2d_MPIAIJ_Fill(DA da,Mat J)
695: {
696: PetscErrorCode ierr;
697: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
698: PetscInt m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
699: PetscInt lstart,lend,pstart,pend,*dnz,*onz;
700: PetscInt ifill_col,*ofill = da->ofill, *dfill = da->dfill;
701: MPI_Comm comm;
702: PetscScalar *values;
703: DAPeriodicType wrap;
704: ISLocalToGlobalMapping ltog;
705: DAStencilType st;
708: /*
709: nc - number of components per grid point
710: col - number of colors needed in one direction for single component problem
711:
712: */
713: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
714: col = 2*s + 1;
715: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
716: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
717: PetscObjectGetComm((PetscObject)da,&comm);
719: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
720: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
721: PetscMalloc(nc*sizeof(PetscInt),&rows);
722: PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
723: DAGetISLocalToGlobalMapping(da,<og);
724:
725: /* determine the matrix preallocation information */
726: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
727: for (i=xs; i<xs+nx; i++) {
729: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
730: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
732: for (j=ys; j<ys+ny; j++) {
733: slot = i - gxs + gnx*(j - gys);
735: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
736: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
738: for (k=0; k<nc; k++) {
739: cnt = 0;
740: for (l=lstart; l<lend+1; l++) {
741: for (p=pstart; p<pend+1; p++) {
742: if (l || p) {
743: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
744: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
745: cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
746: }
747: } else {
748: if (dfill) {
749: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
750: cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
751: } else {
752: for (ifill_col=0; ifill_col<nc; ifill_col++)
753: cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
754: }
755: }
756: }
757: }
758: rows[0] = k + nc*(slot);
759: MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
760: }
761: }
762: }
763: MatSeqAIJSetPreallocation(J,0,dnz);
764: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
765: MatPreallocateFinalize(dnz,onz);
766: MatSetBlockSize(J,nc);
767: MatSetLocalToGlobalMapping(J,ltog);
769: /*
770: For each node in the grid: we get the neighbors in the local (on processor ordering
771: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
772: PETSc ordering.
773: */
774: for (i=xs; i<xs+nx; i++) {
775:
776: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
777: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
778:
779: for (j=ys; j<ys+ny; j++) {
780: slot = i - gxs + gnx*(j - gys);
781:
782: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
783: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
785: for (k=0; k<nc; k++) {
786: cnt = 0;
787: for (l=lstart; l<lend+1; l++) {
788: for (p=pstart; p<pend+1; p++) {
789: if (l || p) {
790: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
791: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
792: cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
793: }
794: } else {
795: if (dfill) {
796: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
797: cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
798: } else {
799: for (ifill_col=0; ifill_col<nc; ifill_col++)
800: cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
801: }
802: }
803: }
804: }
805: rows[0] = k + nc*(slot);
806: MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
807: }
808: }
809: }
810: PetscFree(values);
811: PetscFree(rows);
812: PetscFree(cols);
813: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
814: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
815: return(0);
816: }
818: /* ---------------------------------------------------------------------------------*/
822: PetscErrorCode DAGetMatrix3d_MPIAIJ(DA da,Mat J)
823: {
824: PetscErrorCode ierr;
825: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
826: PetscInt m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
827: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
828: MPI_Comm comm;
829: PetscScalar *values;
830: DAPeriodicType wrap;
831: ISLocalToGlobalMapping ltog;
832: DAStencilType st;
835: /*
836: nc - number of components per grid point
837: col - number of colors needed in one direction for single component problem
838:
839: */
840: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
841: col = 2*s + 1;
843: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
844: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
845: PetscObjectGetComm((PetscObject)da,&comm);
847: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
848: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
849: PetscMalloc(nc*sizeof(PetscInt),&rows);
850: PetscMalloc(col*col*col*nc*sizeof(PetscInt),&cols);
851: DAGetISLocalToGlobalMapping(da,<og);
853: /* determine the matrix preallocation information */
854: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
855: for (i=xs; i<xs+nx; i++) {
856: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
857: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
858: for (j=ys; j<ys+ny; j++) {
859: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
860: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
861: for (k=zs; k<zs+nz; k++) {
862: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
863: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
864:
865: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
866:
867: cnt = 0;
868: for (l=0; l<nc; l++) {
869: for (ii=istart; ii<iend+1; ii++) {
870: for (jj=jstart; jj<jend+1; jj++) {
871: for (kk=kstart; kk<kend+1; kk++) {
872: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
873: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
874: }
875: }
876: }
877: }
878: rows[l] = l + nc*(slot);
879: }
880: MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
881: }
882: }
883: }
884: MatSeqAIJSetPreallocation(J,0,dnz);
885: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
886: MatPreallocateFinalize(dnz,onz);
887: MatSetBlockSize(J,nc);CHKERRQ(ierr)
888: MatSetLocalToGlobalMapping(J,ltog);
890: /*
891: For each node in the grid: we get the neighbors in the local (on processor ordering
892: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
893: PETSc ordering.
894: */
895: for (i=xs; i<xs+nx; i++) {
896: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
897: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
898: for (j=ys; j<ys+ny; j++) {
899: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
900: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
901: for (k=zs; k<zs+nz; k++) {
902: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
903: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
904:
905: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
906:
907: cnt = 0;
908: for (l=0; l<nc; l++) {
909: for (ii=istart; ii<iend+1; ii++) {
910: for (jj=jstart; jj<jend+1; jj++) {
911: for (kk=kstart; kk<kend+1; kk++) {
912: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
913: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
914: }
915: }
916: }
917: }
918: rows[l] = l + nc*(slot);
919: }
920: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
921: }
922: }
923: }
924: PetscFree(values);
925: PetscFree(rows);
926: PetscFree(cols);
927: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
928: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
929: return(0);
930: }
932: /* ---------------------------------------------------------------------------------*/
936: PetscErrorCode DAGetMatrix1d_MPIAIJ(DA da,Mat J)
937: {
938: PetscErrorCode ierr;
939: PetscInt xs,nx,i,i1,slot,gxs,gnx;
940: PetscInt m,dim,s,*cols,nc,*rows,col,cnt,l;
941: PetscInt istart,iend;
942: PetscScalar *values;
943: DAPeriodicType wrap;
944: ISLocalToGlobalMapping ltog;
947: /*
948: nc - number of components per grid point
949: col - number of colors needed in one direction for single component problem
950:
951: */
952: DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
953: col = 2*s + 1;
955: DAGetCorners(da,&xs,0,0,&nx,0,0);
956: DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
958: MatSeqAIJSetPreallocation(J,col*nc,0);
959: MatMPIAIJSetPreallocation(J,col*nc,0,0,0);
960: MatSetBlockSize(J,nc);
961: PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
962: PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
963: PetscMalloc(nc*sizeof(PetscInt),&rows);
964: PetscMalloc(col*nc*sizeof(PetscInt),&cols);
965:
966: DAGetISLocalToGlobalMapping(da,<og);
967: MatSetLocalToGlobalMapping(J,ltog);
968:
969: /*
970: For each node in the grid: we get the neighbors in the local (on processor ordering
971: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
972: PETSc ordering.
973: */
974: for (i=xs; i<xs+nx; i++) {
975: istart = PetscMax(-s,gxs - i);
976: iend = PetscMin(s,gxs + gnx - i - 1);
977: slot = i - gxs;
978:
979: cnt = 0;
980: for (l=0; l<nc; l++) {
981: for (i1=istart; i1<iend+1; i1++) {
982: cols[cnt++] = l + nc*(slot + i1);
983: }
984: rows[l] = l + nc*(slot);
985: }
986: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
987: }
988: PetscFree(values);
989: PetscFree(rows);
990: PetscFree(cols);
991: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
992: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
993: return(0);
994: }
998: PetscErrorCode DAGetMatrix3d_MPIBAIJ(DA da,Mat J)
999: {
1000: PetscErrorCode ierr;
1001: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1002: PetscInt m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1003: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1004: MPI_Comm comm;
1005: PetscScalar *values;
1006: DAPeriodicType wrap;
1007: DAStencilType st;
1008: ISLocalToGlobalMapping ltog;
1011: /*
1012: nc - number of components per grid point
1013: col - number of colors needed in one direction for single component problem
1014:
1015: */
1016: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1017: if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1018: col = 2*s + 1;
1020: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1021: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1022: PetscObjectGetComm((PetscObject)da,&comm);
1024: PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1025: PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1026: PetscMalloc(col*col*col*sizeof(PetscInt),&cols);
1028: DAGetISLocalToGlobalMappingBlck(da,<og);
1030: /* determine the matrix preallocation information */
1031: MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1032: for (i=xs; i<xs+nx; i++) {
1033: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1034: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1035: for (j=ys; j<ys+ny; j++) {
1036: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1037: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1038: for (k=zs; k<zs+nz; k++) {
1039: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1040: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1042: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1044: /* Find block columns in block row */
1045: cnt = 0;
1046: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1047: for (ii=istart; ii<iend+1; ii++) {
1048: for (jj=jstart; jj<jend+1; jj++) {
1049: for (kk=kstart; kk<kend+1; kk++) {
1050: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1051: }
1052: }
1053: }
1054: } else { /* Star stencil */
1055: cnt = 0;
1056: for (ii=istart; ii<iend+1; ii++) {
1057: if (ii) {
1058: /* jj and kk must be zero */
1059: /* cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk; */
1060: cols[cnt++] = slot + ii;
1061: } else {
1062: for (jj=jstart; jj<jend+1; jj++) {
1063: if (jj) {
1064: /* ii and kk must be zero */
1065: cols[cnt++] = slot + gnx*jj;
1066: } else {
1067: /* ii and jj must be zero */
1068: for (kk=kstart; kk<kend+1; kk++) {
1069: cols[cnt++] = slot + gnx*gny*kk;
1070: }
1071: }
1072: }
1073: }
1074: }
1075: }
1076: MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1077: }
1078: }
1079: }
1080: MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1081: MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1082: MatPreallocateFinalize(dnz,onz);
1084: MatSetLocalToGlobalMappingBlock(J,ltog);
1086: /*
1087: For each node in the grid: we get the neighbors in the local (on processor ordering
1088: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1089: PETSc ordering.
1090: */
1092: for (i=xs; i<xs+nx; i++) {
1093: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1094: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1095: for (j=ys; j<ys+ny; j++) {
1096: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1097: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1098: for (k=zs; k<zs+nz; k++) {
1099: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1100: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1101:
1102: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1103:
1104: cnt = 0;
1105: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1106: for (ii=istart; ii<iend+1; ii++) {
1107: for (jj=jstart; jj<jend+1; jj++) {
1108: for (kk=kstart; kk<kend+1; kk++) {
1109: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1110: }
1111: }
1112: }
1113: } else { /* Star stencil */
1114: cnt = 0;
1115: for (ii=istart; ii<iend+1; ii++) {
1116: if (ii) {
1117: /* jj and kk must be zero */
1118: cols[cnt++] = slot + ii;
1119: } else {
1120: for (jj=jstart; jj<jend+1; jj++) {
1121: if (jj) {
1122: /* ii and kk must be zero */
1123: cols[cnt++] = slot + gnx*jj;
1124: } else {
1125: /* ii and jj must be zero */
1126: for (kk=kstart; kk<kend+1; kk++) {
1127: cols[cnt++] = slot + gnx*gny*kk;
1128: }
1129: }
1130: }
1131: }
1132: }
1133: }
1134: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1135: }
1136: }
1137: }
1138: PetscFree(values);
1139: PetscFree(cols);
1140: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1141: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1142: return(0);
1143: }
1145: /* BAD! Almost identical to the BAIJ one */
1148: PetscErrorCode DAGetMatrix3d_MPISBAIJ(DA da,Mat J)
1149: {
1150: PetscErrorCode ierr;
1151: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1152: PetscInt m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1153: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1154: MPI_Comm comm;
1155: PetscScalar *values;
1156: DAPeriodicType wrap;
1157: DAStencilType st;
1158: ISLocalToGlobalMapping ltog;
1161: /*
1162: nc - number of components per grid point
1163: col - number of colors needed in one direction for single component problem
1164:
1165: */
1166: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1167: if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1168: col = 2*s + 1;
1170: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1171: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1172: PetscObjectGetComm((PetscObject)da,&comm);
1174: /* create the matrix */
1175: PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1176: PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1177: PetscMalloc(col*col*col*sizeof(PetscInt),&cols);
1179: DAGetISLocalToGlobalMappingBlck(da,<og);
1181: /* determine the matrix preallocation information */
1182: MatPreallocateSymmetricInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1183: for (i=xs; i<xs+nx; i++) {
1184: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1185: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1186: for (j=ys; j<ys+ny; j++) {
1187: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1188: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1189: for (k=zs; k<zs+nz; k++) {
1190: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1191: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1193: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1195: /* Find block columns in block row */
1196: cnt = 0;
1197: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1198: for (ii=istart; ii<iend+1; ii++) {
1199: for (jj=jstart; jj<jend+1; jj++) {
1200: for (kk=kstart; kk<kend+1; kk++) {
1201: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1202: }
1203: }
1204: }
1205: } else { /* Star stencil */
1206: cnt = 0;
1207: for (ii=istart; ii<iend+1; ii++) {
1208: if (ii) {
1209: /* jj and kk must be zero */
1210: /* cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk; */
1211: cols[cnt++] = slot + ii;
1212: } else {
1213: for (jj=jstart; jj<jend+1; jj++) {
1214: if (jj) {
1215: /* ii and kk must be zero */
1216: cols[cnt++] = slot + gnx*jj;
1217: } else {
1218: /* ii and jj must be zero */
1219: for (kk=kstart; kk<kend+1; kk++) {
1220: cols[cnt++] = slot + gnx*gny*kk;
1221: }
1222: }
1223: }
1224: }
1225: }
1226: }
1227: MatPreallocateSymmetricSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1228: }
1229: }
1230: }
1231: MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1232: MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1233: MatPreallocateFinalize(dnz,onz);
1235: MatSetLocalToGlobalMappingBlock(J,ltog);
1237: /*
1238: For each node in the grid: we get the neighbors in the local (on processor ordering
1239: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1240: PETSc ordering.
1241: */
1243: for (i=xs; i<xs+nx; i++) {
1244: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1245: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1246: for (j=ys; j<ys+ny; j++) {
1247: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1248: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1249: for (k=zs; k<zs+nz; k++) {
1250: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1251: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1252:
1253: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1254:
1255: cnt = 0;
1256: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1257: for (ii=istart; ii<iend+1; ii++) {
1258: for (jj=jstart; jj<jend+1; jj++) {
1259: for (kk=kstart; kk<kend+1; kk++) {
1260: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1261: }
1262: }
1263: }
1264: } else { /* Star stencil */
1265: cnt = 0;
1266: for (ii=istart; ii<iend+1; ii++) {
1267: if (ii) {
1268: /* jj and kk must be zero */
1269: cols[cnt++] = slot + ii;
1270: } else {
1271: for (jj=jstart; jj<jend+1; jj++) {
1272: if (jj) {
1273: /* ii and kk must be zero */
1274: cols[cnt++] = slot + gnx*jj;
1275: } else {
1276: /* ii and jj must be zero */
1277: for (kk=kstart; kk<kend+1; kk++) {
1278: cols[cnt++] = slot + gnx*gny*kk;
1279: }
1280: }
1281: }
1282: }
1283: }
1284: }
1285: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1286: }
1287: }
1288: }
1289: PetscFree(values);
1290: PetscFree(cols);
1291: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1292: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1293: return(0);
1294: }
1297: /* ---------------------------------------------------------------------------------*/
1301: PetscErrorCode DAGetMatrix3d_MPIAIJ_Fill(DA da,Mat J)
1302: {
1303: PetscErrorCode ierr;
1304: PetscInt xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1305: PetscInt m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
1306: PetscInt istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1307: PetscInt ifill_col,*dfill = da->dfill,*ofill = da->ofill;
1308: MPI_Comm comm;
1309: PetscScalar *values;
1310: DAPeriodicType wrap;
1311: ISLocalToGlobalMapping ltog;
1312: DAStencilType st;
1315: /*
1316: nc - number of components per grid point
1317: col - number of colors needed in one direction for single component problem
1318:
1319: */
1320: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1321: col = 2*s + 1;
1322: if (DAXPeriodic(wrap) && (m % col)){
1323: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1324: by 2*stencil_width + 1\n");
1325: }
1326: if (DAYPeriodic(wrap) && (n % col)){
1327: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1328: by 2*stencil_width + 1\n");
1329: }
1330: if (DAZPeriodic(wrap) && (p % col)){
1331: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1332: by 2*stencil_width + 1\n");
1333: }
1335: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1336: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1337: PetscObjectGetComm((PetscObject)da,&comm);
1339: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1340: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1341: PetscMalloc(nc*sizeof(PetscInt),&rows);
1342: PetscMalloc(col*col*col*nc*sizeof(PetscInt),&cols);
1343: DAGetISLocalToGlobalMapping(da,<og);
1345: /* determine the matrix preallocation information */
1346: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1349: for (i=xs; i<xs+nx; i++) {
1350: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1351: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1352: for (j=ys; j<ys+ny; j++) {
1353: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1354: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1355: for (k=zs; k<zs+nz; k++) {
1356: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1357: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1358:
1359: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1360:
1361: for (l=0; l<nc; l++) {
1362: cnt = 0;
1363: for (ii=istart; ii<iend+1; ii++) {
1364: for (jj=jstart; jj<jend+1; jj++) {
1365: for (kk=kstart; kk<kend+1; kk++) {
1366: if (ii || jj || kk) {
1367: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1368: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1369: cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1370: }
1371: } else {
1372: if (dfill) {
1373: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1374: cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1375: } else {
1376: for (ifill_col=0; ifill_col<nc; ifill_col++)
1377: cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1378: }
1379: }
1380: }
1381: }
1382: }
1383: rows[0] = l + nc*(slot);
1384: MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
1385: }
1386: }
1387: }
1388: }
1389: MatSeqAIJSetPreallocation(J,0,dnz);
1390: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1391: MatPreallocateFinalize(dnz,onz);
1392: MatSetBlockSize(J,nc);
1393: MatSetLocalToGlobalMapping(J,ltog);
1395: /*
1396: For each node in the grid: we get the neighbors in the local (on processor ordering
1397: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1398: PETSc ordering.
1399: */
1400: for (i=xs; i<xs+nx; i++) {
1401: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1402: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1403: for (j=ys; j<ys+ny; j++) {
1404: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1405: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1406: for (k=zs; k<zs+nz; k++) {
1407: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1408: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1409:
1410: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1411:
1412: for (l=0; l<nc; l++) {
1413: cnt = 0;
1414: for (ii=istart; ii<iend+1; ii++) {
1415: for (jj=jstart; jj<jend+1; jj++) {
1416: for (kk=kstart; kk<kend+1; kk++) {
1417: if (ii || jj || kk) {
1418: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1419: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1420: cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1421: }
1422: } else {
1423: if (dfill) {
1424: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1425: cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1426: } else {
1427: for (ifill_col=0; ifill_col<nc; ifill_col++)
1428: cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1429: }
1430: }
1431: }
1432: }
1433: }
1434: rows[0] = l + nc*(slot);
1435: MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
1436: }
1437: }
1438: }
1439: }
1440: PetscFree(values);
1441: PetscFree(rows);
1442: PetscFree(cols);
1443: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1444: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1445: return(0);
1446: }