Actual source code: fdda.c
1: /*$Id: fdda.c,v 1.75 2001/08/07 21:31:51 bsmith Exp $*/
2:
3: #include src/dm/da/daimpl.h
4: #include petscmat.h
7: EXTERN int DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
8: EXTERN int DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
9: EXTERN int DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
10: EXTERN int DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);
12: /*
13: For ghost i that may be negative or greater than the upper bound this
14: maps it into the 0:m-1 range using periodicity
15: */
16: #define SetInRange(i,m) ((i < 0) ? m+i:((i >= m) ? i-m:i))
20: static int DASetBlockFills_Private(int *dfill,int w,int **rfill)
21: {
22: int ierr,i,j,nz,*fill;
25: /* count number nonzeros */
26: nz = 0;
27: for (i=0; i<w; i++) {
28: for (j=0; j<w; j++) {
29: if (dfill[w*i+j]) nz++;
30: }
31: }
32: PetscMalloc((nz + w + 1)*sizeof(int),&fill);
33: /* construct modified CSR storage of nonzero structure */
34: nz = w + 1;
35: for (i=0; i<w; i++) {
36: fill[i] = nz;
37: for (j=0; j<w; j++) {
38: if (dfill[w*i+j]) {
39: fill[nz] = j;
40: nz++;
41: }
42: }
43: }
44: fill[w] = nz;
45:
46: *rfill = fill;
47: return(0);
48: }
52: /*@C
53: DASetBlockFills - Sets the fill pattern in each block for a multi-component problem
54: of the matrix returned by DAGetMatrix().
56: Collective on DA
58: Input Parameter:
59: + da - the distributed array
60: . dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
61: - ofill - the fill pattern in the off-diagonal blocks
64: Level: developer
66: Notes: This only makes sense when you are doing multicomponent problems but using the
67: MPIAIJ matrix format
69: The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
70: representing coupling and 0 entries for missing coupling. For example
71: $ dfill[3][3] = {1, 0, 0,
72: $ 1, 1, 0,
73: $ 0, 1, 1}
74: means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with
75: itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the
76: diagonal block.
78: DASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
79: can be represented in the dfill, ofill format
81: Contributed by Glenn Hammond
83: .seealso DAGetMatrix(), DASetGetMatrix()
85: @*/
86: int DASetBlockFills(DA da,int *dfill,int *ofill)
87: {
91: if (dfill) {
92: DASetBlockFills_Private(dfill,da->w,&da->dfill);
93: }
94: DASetBlockFills_Private(ofill,da->w,&da->ofill);
95: return(0);
96: }
101: /*@C
102: DAGetColoring - Gets the coloring required for computing the Jacobian via
103: finite differences on a function defined using a stencil on the DA.
105: Collective on DA
107: Input Parameter:
108: + da - the distributed array
109: - ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED
111: Output Parameters:
112: . coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)
114: Level: advanced
116: Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used
117: for efficient (parallel or thread based) triangular solves etc is NOT yet
118: available.
121: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring
123: @*/
124: int DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
125: {
126: int ierr,dim;
129: /*
130: m
131: ------------------------------------------------------
132: | |
133: | |
134: | ---------------------- |
135: | | | |
136: n | yn | | |
137: | | | |
138: | .--------------------- |
139: | (xs,ys) xn |
140: | . |
141: | (gxs,gys) |
142: | |
143: -----------------------------------------------------
144: */
146: /*
147: nc - number of components per grid point
148: col - number of colors needed in one direction for single component problem
149:
150: */
151: DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);
153: /*
154: We do not provide a getcoloring function in the DA operations because
155: the basic DA does not know about matrices. We think of DA as being more
156: more low-level then matrices.
157: */
158: if (dim == 1) {
159: DAGetColoring1d_MPIAIJ(da,ctype,coloring);
160: } else if (dim == 2) {
161: DAGetColoring2d_MPIAIJ(da,ctype,coloring);
162: } else if (dim == 3) {
163: DAGetColoring3d_MPIAIJ(da,ctype,coloring);
164: } else {
165: SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
166: }
167: return(0);
168: }
170: /* ---------------------------------------------------------------------------------*/
174: int DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
175: {
176: int ierr,xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny;
177: int m,n,M,N,dim,s,k,nc,col,size;
178: MPI_Comm comm;
179: DAPeriodicType wrap;
180: DAStencilType st;
181: ISColoringValue *colors;
184: /*
185: nc - number of components per grid point
186: col - number of colors needed in one direction for single component problem
187:
188: */
189: DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&wrap,&st);
190: col = 2*s + 1;
191: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
192: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
193: PetscObjectGetComm((PetscObject)da,&comm);
194: MPI_Comm_size(comm,&size);
196: /* special case as taught to us by Paul Hovland */
197: if (st == DA_STENCIL_STAR && s == 1) {
198: DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
199: } else {
201: if (DAXPeriodic(wrap) && (m % col)){
202: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
203: by 2*stencil_width + 1\n");
204: }
205: if (DAYPeriodic(wrap) && (n % col)){
206: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
207: by 2*stencil_width + 1\n");
208: }
209: if (ctype == IS_COLORING_LOCAL) {
210: if (!da->localcoloring) {
211: PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
212: ii = 0;
213: for (j=ys; j<ys+ny; j++) {
214: for (i=xs; i<xs+nx; i++) {
215: for (k=0; k<nc; k++) {
216: colors[ii++] = k + nc*((i % col) + col*(j % col));
217: }
218: }
219: }
220: ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
221: }
222: *coloring = da->localcoloring;
223: } else if (ctype == IS_COLORING_GHOSTED) {
224: if (!da->ghostedcoloring) {
225: PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
226: ii = 0;
227: for (j=gys; j<gys+gny; j++) {
228: for (i=gxs; i<gxs+gnx; i++) {
229: for (k=0; k<nc; k++) {
230: /* the complicated stuff is to handle periodic boundaries */
231: colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
232: }
233: }
234: }
235: ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
236: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
237: }
238: *coloring = da->ghostedcoloring;
239: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
240: }
241: ISColoringReference(*coloring);
242: return(0);
243: }
245: /* ---------------------------------------------------------------------------------*/
249: int DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
250: {
251: int ierr,xs,ys,nx,ny,i,j,gxs,gys,gnx,gny;
252: int m,n,p,dim,s,k,nc,col,size,zs,gzs,ii,l,nz,gnz,M,N,P;
253: MPI_Comm comm;
254: DAPeriodicType wrap;
255: DAStencilType st;
256: ISColoringValue *colors;
259: /*
260: nc - number of components per grid point
261: col - number of colors needed in one direction for single component problem
262:
263: */
264: DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
265: col = 2*s + 1;
266: if (DAXPeriodic(wrap) && (m % col)){
267: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
268: by 2*stencil_width + 1\n");
269: }
270: if (DAYPeriodic(wrap) && (n % col)){
271: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
272: by 2*stencil_width + 1\n");
273: }
274: if (DAZPeriodic(wrap) && (p % col)){
275: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
276: by 2*stencil_width + 1\n");
277: }
279: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
280: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
281: PetscObjectGetComm((PetscObject)da,&comm);
282: MPI_Comm_size(comm,&size);
284: /* create the coloring */
285: if (ctype == IS_COLORING_LOCAL) {
286: if (!da->localcoloring) {
287: PetscMalloc(nc*nx*ny*nz*sizeof(ISColoringValue),&colors);
288: ii = 0;
289: for (k=zs; k<zs+nz; k++) {
290: for (j=ys; j<ys+ny; j++) {
291: for (i=xs; i<xs+nx; i++) {
292: for (l=0; l<nc; l++) {
293: colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
294: }
295: }
296: }
297: }
298: ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
299: }
300: *coloring = da->localcoloring;
301: } else if (ctype == IS_COLORING_GHOSTED) {
302: if (!da->ghostedcoloring) {
303: PetscMalloc(nc*gnx*gny*gnz*sizeof(ISColoringValue),&colors);
304: ii = 0;
305: for (k=gzs; k<gzs+gnz; k++) {
306: for (j=gys; j<gys+gny; j++) {
307: for (i=gxs; i<gxs+gnx; i++) {
308: for (l=0; l<nc; l++) {
309: /* the complicated stuff is to handle periodic boundaries */
310: colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
311: }
312: }
313: }
314: }
315: ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
316: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
317: }
318: *coloring = da->ghostedcoloring;
319: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
320: ISColoringReference(*coloring);
321: return(0);
322: }
324: /* ---------------------------------------------------------------------------------*/
328: int DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
329: {
330: int ierr,xs,nx,i,i1,gxs,gnx,l;
331: int m,M,dim,s,nc,col,size;
332: MPI_Comm comm;
333: DAPeriodicType wrap;
334: ISColoringValue *colors;
337: /*
338: nc - number of components per grid point
339: col - number of colors needed in one direction for single component problem
340:
341: */
342: DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
343: col = 2*s + 1;
345: if (DAXPeriodic(wrap) && (m % col)) {
346: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisible\n\
347: by 2*stencil_width + 1\n");
348: }
350: DAGetCorners(da,&xs,0,0,&nx,0,0);
351: DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
352: PetscObjectGetComm((PetscObject)da,&comm);
353: MPI_Comm_size(comm,&size);
355: /* create the coloring */
356: if (ctype == IS_COLORING_LOCAL) {
357: if (!da->localcoloring) {
358: PetscMalloc(nc*nx*sizeof(ISColoringValue),&colors);
359: i1 = 0;
360: for (i=xs; i<xs+nx; i++) {
361: for (l=0; l<nc; l++) {
362: colors[i1++] = l + nc*(i % col);
363: }
364: }
365: ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
366: }
367: *coloring = da->localcoloring;
368: } else if (ctype == IS_COLORING_GHOSTED) {
369: if (!da->ghostedcoloring) {
370: PetscMalloc(nc*gnx*sizeof(ISColoringValue),&colors);
371: i1 = 0;
372: for (i=gxs; i<gxs+gnx; i++) {
373: for (l=0; l<nc; l++) {
374: /* the complicated stuff is to handle periodic boundaries */
375: colors[i1++] = l + nc*(SetInRange(i,m) % col);
376: }
377: }
378: ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
379: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
380: }
381: *coloring = da->ghostedcoloring;
382: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
383: ISColoringReference(*coloring);
384: return(0);
385: }
389: int DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
390: {
391: int ierr,xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny;
392: int m,n,dim,s,k,nc;
393: MPI_Comm comm;
394: DAPeriodicType wrap;
395: ISColoringValue *colors;
398: /*
399: nc - number of components per grid point
400: col - number of colors needed in one direction for single component problem
401:
402: */
403: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,0);
404: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
405: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
406: PetscObjectGetComm((PetscObject)da,&comm);
408: if (DAXPeriodic(wrap) && (m % 5)){
409: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
410: by 5\n");
411: }
412: if (DAYPeriodic(wrap) && (n % 5)){
413: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
414: by 5\n");
415: }
417: /* create the coloring */
418: if (ctype == IS_COLORING_LOCAL) {
419: if (!da->localcoloring) {
420: PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
421: ii = 0;
422: for (j=ys; j<ys+ny; j++) {
423: for (i=xs; i<xs+nx; i++) {
424: for (k=0; k<nc; k++) {
425: colors[ii++] = k + nc*((3*j+i) % 5);
426: }
427: }
428: }
429: ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
430: }
431: *coloring = da->localcoloring;
432: } else if (ctype == IS_COLORING_GHOSTED) {
433: if (!da->ghostedcoloring) {
434: PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
435: ii = 0;
436: for (j=gys; j<gys+gny; j++) {
437: for (i=gxs; i<gxs+gnx; i++) {
438: for (k=0; k<nc; k++) {
439: colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
440: }
441: }
442: }
443: ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
444: ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
445: }
446: *coloring = da->ghostedcoloring;
447: } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
448: return(0);
449: }
451: /* =========================================================================== */
452: EXTERN int DAGetMatrix1d_MPIAIJ(DA,Mat);
453: EXTERN int DAGetMatrix2d_MPIAIJ(DA,Mat);
454: EXTERN int DAGetMatrix2d_MPIAIJ_Fill(DA,Mat);
455: EXTERN int DAGetMatrix3d_MPIAIJ(DA,Mat);
456: EXTERN int DAGetMatrix3d_MPIAIJ_Fill(DA,Mat);
457: EXTERN int DAGetMatrix3d_MPIBAIJ(DA,Mat);
458: EXTERN int DAGetMatrix3d_MPISBAIJ(DA,Mat);
462: /*@C
463: DAGetMatrix - Creates a matrix with the correct parallel layout and nonzero structure required for
464: computing the Jacobian on a function defined using the stencil set in the DA.
466: Collective on DA
468: Input Parameter:
469: + da - the distributed array
470: - mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, or
471: any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).
473: Output Parameters:
474: . J - matrix with the correct nonzero structure
475: (obviously without the correct Jacobian values)
477: Level: advanced
479: Notes: This properly preallocates the number of nonzeros in the sparse matrix so you
480: do not need to do it yourself.
482: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()
484: @*/
485: int DAGetMatrix(DA da,MatType mtype,Mat *J)
486: {
487: int ierr,dim,dof,nx,ny,nz;
488: Mat A;
489: MPI_Comm comm;
490: MatType Atype;
491: void (*aij)(void)=PETSC_NULL,(*baij)(void)=PETSC_NULL,(*sbaij)(void)=PETSC_NULL;
494: /*
495: m
496: ------------------------------------------------------
497: | |
498: | |
499: | ---------------------- |
500: | | | |
501: n | yn | | |
502: | | | |
503: | .--------------------- |
504: | (xs,ys) xn |
505: | . |
506: | (gxs,gys) |
507: | |
508: -----------------------------------------------------
509: */
511: /*
512: nc - number of components per grid point
513: col - number of colors needed in one direction for single component problem
514:
515: */
516: DAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0);
517: DAGetCorners(da,0,0,0,&nx,&ny,&nz);
518: PetscObjectGetComm((PetscObject)da,&comm);
519: MatCreate(comm,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,&A);
520: MatSetType(A,mtype);
521: MatSetFromOptions(A);
522: MatGetType(A,&Atype);
523: /*
524: We do not provide a getmatrix function in the DA operations because
525: the basic DA does not know about matrices. We think of DA as being more
526: more low-level then matrices. This is kind of cheating but, cause sometimes
527: we think of DA has higher level then matrices.
529: We could switch based on Atype (or mtype), but we do not since the
530: specialized setting routines depend only the particular preallocation
531: details of the matrix, not the type itself.
532: */
533: PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
534: if (!aij) {
535: PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
536: }
537: if (aij) {
538: if (dim == 1) {
539: DAGetMatrix1d_MPIAIJ(da,A);
540: } else if (dim == 2) {
541: if (da->ofill) {
542: DAGetMatrix2d_MPIAIJ_Fill(da,A);
543: } else {
544: DAGetMatrix2d_MPIAIJ(da,A);
545: }
546: } else if (dim == 3) {
547: if (da->ofill) {
548: DAGetMatrix3d_MPIAIJ_Fill(da,A);
549: } else {
550: DAGetMatrix3d_MPIAIJ(da,A);
551: }
552: }
553: } else if (dim == 3) {
554: PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
555: if (!baij) {
556: PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
557: }
558: if (baij) {
559: DAGetMatrix3d_MPIBAIJ(da,A);
560: } else {
561: PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
562: if (!sbaij) {
563: PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
564: }
565: if (sbaij) {
566: DAGetMatrix3d_MPISBAIJ(da,A);
567: } else {
568: SETERRQ2(PETSC_ERR_SUP,"Not implemented for %d dimension and Matrix Type: %s!\n" \
569: "Send mail to petsc-maint@mcs.anl.gov for code",
570: dim,Atype);
571: }
572: }
573: } else {
574: SETERRQ2(PETSC_ERR_SUP,"Not implemented for %d dimension and matrix type: %s!\n" \
575: "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype);
576: }
577: *J = A;
578: return(0);
579: }
581: /* ---------------------------------------------------------------------------------*/
584: int DAGetMatrix2d_MPIAIJ(DA da,Mat J)
585: {
586: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
587: int m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
588: int lstart,lend,pstart,pend,*dnz,*onz;
589: int dims[2],starts[2];
590: MPI_Comm comm;
591: PetscScalar *values;
592: DAPeriodicType wrap;
593: ISLocalToGlobalMapping ltog;
594: DAStencilType st;
597: /*
598: nc - number of components per grid point
599: col - number of colors needed in one direction for single component problem
600:
601: */
602: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
603: col = 2*s + 1;
604: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
605: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
606: PetscObjectGetComm((PetscObject)da,&comm);
608: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
609: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
610: PetscMalloc(nc*sizeof(int),&rows);
611: PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
612: DAGetISLocalToGlobalMapping(da,<og);
613:
614: /* determine the matrix preallocation information */
615: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
616: for (i=xs; i<xs+nx; i++) {
618: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
619: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
621: for (j=ys; j<ys+ny; j++) {
622: slot = i - gxs + gnx*(j - gys);
624: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
625: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
627: cnt = 0;
628: for (k=0; k<nc; k++) {
629: for (l=lstart; l<lend+1; l++) {
630: for (p=pstart; p<pend+1; p++) {
631: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
632: cols[cnt++] = k + nc*(slot + gnx*l + p);
633: }
634: }
635: }
636: rows[k] = k + nc*(slot);
637: }
638: MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
639: }
640: }
641: MatSeqAIJSetPreallocation(J,0,dnz);
642: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
643: MatPreallocateFinalize(dnz,onz);
645: MatSetLocalToGlobalMapping(J,ltog);
646: DAGetGhostCorners(da,&starts[0],&starts[1],PETSC_IGNORE,&dims[0],&dims[1],PETSC_IGNORE);
647: MatSetStencil(J,2,dims,starts,nc);
649: /*
650: For each node in the grid: we get the neighbors in the local (on processor ordering
651: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
652: PETSc ordering.
653: */
654: for (i=xs; i<xs+nx; i++) {
655:
656: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
657: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
658:
659: for (j=ys; j<ys+ny; j++) {
660: slot = i - gxs + gnx*(j - gys);
661:
662: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
663: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
665: cnt = 0;
666: for (k=0; k<nc; k++) {
667: for (l=lstart; l<lend+1; l++) {
668: for (p=pstart; p<pend+1; p++) {
669: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star have either l = 0 or p = 0 */
670: cols[cnt++] = k + nc*(slot + gnx*l + p);
671: }
672: }
673: }
674: rows[k] = k + nc*(slot);
675: }
676: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
677: }
678: }
679: PetscFree(values);
680: PetscFree(rows);
681: PetscFree(cols);
682: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
683: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
684: return(0);
685: }
689: int DAGetMatrix2d_MPIAIJ_Fill(DA da,Mat J)
690: {
691: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
692: int m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
693: int lstart,lend,pstart,pend,*dnz,*onz;
694: int dims[2],starts[2],ifill_col,*ofill = da->ofill, *dfill = da->dfill;
695: MPI_Comm comm;
696: PetscScalar *values;
697: DAPeriodicType wrap;
698: ISLocalToGlobalMapping ltog;
699: DAStencilType st;
702: /*
703: nc - number of components per grid point
704: col - number of colors needed in one direction for single component problem
705:
706: */
707: DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
708: col = 2*s + 1;
709: DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
710: DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
711: PetscObjectGetComm((PetscObject)da,&comm);
713: PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
714: PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
715: PetscMalloc(nc*sizeof(int),&rows);
716: PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
717: DAGetISLocalToGlobalMapping(da,<og);
718:
719: /* determine the matrix preallocation information */
720: MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
721: for (i=xs; i<xs+nx; i++) {
723: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
724: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
726: for (j=ys; j<ys+ny; j++) {
727: slot = i - gxs + gnx*(j - gys);
729: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
730: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
732: for (k=0; k<nc; k++) {
733: cnt = 0;
734: for (l=lstart; l<lend+1; l++) {
735: for (p=pstart; p<pend+1; p++) {
736: if (l || p) {
737: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
738: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
739: cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
740: }
741: } else {
742: if (dfill) {
743: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
744: cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
745: } else {
746: for (ifill_col=0; ifill_col<nc; ifill_col++)
747: cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
748: }
749: }
750: }
751: }
752: rows[0] = k + nc*(slot);
753: MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
754: }
755: }
756: }
757: MatSeqAIJSetPreallocation(J,0,dnz);
758: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
759: MatPreallocateFinalize(dnz,onz);
761: MatSetLocalToGlobalMapping(J,ltog);
762: DAGetGhostCorners(da,&starts[0],&starts[1],PETSC_IGNORE,&dims[0],&dims[1],PETSC_IGNORE);
763: MatSetStencil(J,2,dims,starts,nc);
765: /*
766: For each node in the grid: we get the neighbors in the local (on processor ordering
767: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
768: PETSc ordering.
769: */
770: for (i=xs; i<xs+nx; i++) {
771:
772: pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
773: pend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
774:
775: for (j=ys; j<ys+ny; j++) {
776: slot = i - gxs + gnx*(j - gys);
777:
778: lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
779: lend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
781: for (k=0; k<nc; k++) {
782: cnt = 0;
783: for (l=lstart; l<lend+1; l++) {
784: for (p=pstart; p<pend+1; p++) {
785: if (l || p) {
786: if ((st == DA_STENCIL_BOX) || (!l || !p)) { /* entries on star */
787: for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
788: cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
789: }
790: } else {
791: if (dfill) {
792: for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
793: cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
794: } else {
795: for (ifill_col=0; ifill_col<nc; ifill_col++)
796: cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
797: }
798: }
799: }
800: }
801: rows[0] = k + nc*(slot);
802: MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
803: }
804: }
805: }
806: PetscFree(values);
807: PetscFree(rows);
808: PetscFree(cols);
809: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
810: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
811: return(0);
812: }
814: /* ---------------------------------------------------------------------------------*/
818: int DAGetMatrix3d_MPIAIJ(DA da,Mat J)
819: {
820: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
821: int m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
822: int istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
823: int dims[3],starts[3];
824: MPI_Comm comm;
825: PetscScalar *values;
826: DAPeriodicType wrap;
827: ISLocalToGlobalMapping ltog;
828: DAStencilType st;
831: /*
832: nc - number of components per grid point
833: col - number of colors needed in one direction for single component problem
834:
835: */
836: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
837: col = 2*s + 1;
839: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
840: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
841: PetscObjectGetComm((PetscObject)da,&comm);
843: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
844: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
845: PetscMalloc(nc*sizeof(int),&rows);
846: PetscMalloc(col*col*col*nc*sizeof(int),&cols);
847: DAGetISLocalToGlobalMapping(da,<og);
849: /* determine the matrix preallocation information */
850: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
851: for (i=xs; i<xs+nx; i++) {
852: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
853: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
854: for (j=ys; j<ys+ny; j++) {
855: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
856: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
857: for (k=zs; k<zs+nz; k++) {
858: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
859: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
860:
861: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
862:
863: cnt = 0;
864: for (l=0; l<nc; l++) {
865: for (ii=istart; ii<iend+1; ii++) {
866: for (jj=jstart; jj<jend+1; jj++) {
867: for (kk=kstart; kk<kend+1; kk++) {
868: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
869: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
870: }
871: }
872: }
873: }
874: rows[l] = l + nc*(slot);
875: }
876: MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
877: }
878: }
879: }
880: MatSeqAIJSetPreallocation(J,0,dnz);
881: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
882: MatPreallocateFinalize(dnz,onz);
884: MatSetLocalToGlobalMapping(J,ltog);
885: DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
886: MatSetStencil(J,3,dims,starts,nc);
888: /*
889: For each node in the grid: we get the neighbors in the local (on processor ordering
890: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
891: PETSc ordering.
892: */
893: for (i=xs; i<xs+nx; i++) {
894: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
895: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
896: for (j=ys; j<ys+ny; j++) {
897: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
898: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
899: for (k=zs; k<zs+nz; k++) {
900: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
901: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
902:
903: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
904:
905: cnt = 0;
906: for (l=0; l<nc; l++) {
907: for (ii=istart; ii<iend+1; ii++) {
908: for (jj=jstart; jj<jend+1; jj++) {
909: for (kk=kstart; kk<kend+1; kk++) {
910: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
911: cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
912: }
913: }
914: }
915: }
916: rows[l] = l + nc*(slot);
917: }
918: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
919: }
920: }
921: }
922: PetscFree(values);
923: PetscFree(rows);
924: PetscFree(cols);
925: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
926: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
927: return(0);
928: }
930: /* ---------------------------------------------------------------------------------*/
934: int DAGetMatrix1d_MPIAIJ(DA da,Mat J)
935: {
936: int ierr,xs,nx,i,i1,slot,gxs,gnx;
937: int m,dim,s,*cols,nc,*rows,col,cnt,l;
938: int istart,iend;
939: int dims[1],starts[1];
940: PetscScalar *values;
941: DAPeriodicType wrap;
942: ISLocalToGlobalMapping ltog;
945: /*
946: nc - number of components per grid point
947: col - number of colors needed in one direction for single component problem
948:
949: */
950: DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
951: col = 2*s + 1;
953: DAGetCorners(da,&xs,0,0,&nx,0,0);
954: DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
956: MatSeqAIJSetPreallocation(J,col*nc,0);
957: MatMPIAIJSetPreallocation(J,col*nc,0,0,0);
959: DAGetGhostCorners(da,&starts[0],PETSC_IGNORE,PETSC_IGNORE,&dims[0],PETSC_IGNORE,PETSC_IGNORE);
960: MatSetStencil(J,1,dims,starts,nc);
961:
962: PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
963: PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
964: PetscMalloc(nc*sizeof(int),&rows);
965: PetscMalloc(col*nc*sizeof(int),&cols);
966:
967: DAGetISLocalToGlobalMapping(da,<og);
968: MatSetLocalToGlobalMapping(J,ltog);
969:
970: /*
971: For each node in the grid: we get the neighbors in the local (on processor ordering
972: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
973: PETSc ordering.
974: */
975: for (i=xs; i<xs+nx; i++) {
976: istart = PetscMax(-s,gxs - i);
977: iend = PetscMin(s,gxs + gnx - i - 1);
978: slot = i - gxs;
979:
980: cnt = 0;
981: for (l=0; l<nc; l++) {
982: for (i1=istart; i1<iend+1; i1++) {
983: cols[cnt++] = l + nc*(slot + i1);
984: }
985: rows[l] = l + nc*(slot);
986: }
987: MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
988: }
989: PetscFree(values);
990: PetscFree(rows);
991: PetscFree(cols);
992: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
993: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
994: return(0);
995: }
999: int DAGetMatrix3d_MPIBAIJ(DA da,Mat J)
1000: {
1001: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1002: int m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1003: int istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1004: MPI_Comm comm;
1005: PetscScalar *values;
1006: DAPeriodicType wrap;
1007: DAStencilType st;
1008: ISLocalToGlobalMapping ltog;
1011: /*
1012: nc - number of components per grid point
1013: col - number of colors needed in one direction for single component problem
1014:
1015: */
1016: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1017: if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1018: col = 2*s + 1;
1020: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1021: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1022: PetscObjectGetComm((PetscObject)da,&comm);
1024: PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1025: PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1026: PetscMalloc(col*col*col*sizeof(int),&cols);
1028: DAGetISLocalToGlobalMappingBlck(da,<og);
1030: /* determine the matrix preallocation information */
1031: MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1032: for (i=xs; i<xs+nx; i++) {
1033: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1034: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1035: for (j=ys; j<ys+ny; j++) {
1036: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1037: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1038: for (k=zs; k<zs+nz; k++) {
1039: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1040: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1042: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1044: /* Find block columns in block row */
1045: cnt = 0;
1046: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1047: for (ii=istart; ii<iend+1; ii++) {
1048: for (jj=jstart; jj<jend+1; jj++) {
1049: for (kk=kstart; kk<kend+1; kk++) {
1050: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1051: }
1052: }
1053: }
1054: } else { /* Star stencil */
1055: cnt = 0;
1056: for (ii=istart; ii<iend+1; ii++) {
1057: if (ii) {
1058: /* jj and kk must be zero */
1059: /* cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk; */
1060: cols[cnt++] = slot + ii;
1061: } else {
1062: for (jj=jstart; jj<jend+1; jj++) {
1063: if (jj) {
1064: /* ii and kk must be zero */
1065: cols[cnt++] = slot + gnx*jj;
1066: } else {
1067: /* ii and jj must be zero */
1068: for (kk=kstart; kk<kend+1; kk++) {
1069: cols[cnt++] = slot + gnx*gny*kk;
1070: }
1071: }
1072: }
1073: }
1074: }
1075: }
1076: MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1077: }
1078: }
1079: }
1080: MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1081: MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1082: MatPreallocateFinalize(dnz,onz);
1084: MatSetLocalToGlobalMappingBlock(J,ltog);
1086: /*
1087: For each node in the grid: we get the neighbors in the local (on processor ordering
1088: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1089: PETSc ordering.
1090: */
1092: for (i=xs; i<xs+nx; i++) {
1093: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1094: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1095: for (j=ys; j<ys+ny; j++) {
1096: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1097: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1098: for (k=zs; k<zs+nz; k++) {
1099: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1100: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1101:
1102: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1103:
1104: cnt = 0;
1105: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1106: for (ii=istart; ii<iend+1; ii++) {
1107: for (jj=jstart; jj<jend+1; jj++) {
1108: for (kk=kstart; kk<kend+1; kk++) {
1109: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1110: }
1111: }
1112: }
1113: } else { /* Star stencil */
1114: cnt = 0;
1115: for (ii=istart; ii<iend+1; ii++) {
1116: if (ii) {
1117: /* jj and kk must be zero */
1118: cols[cnt++] = slot + ii;
1119: } else {
1120: for (jj=jstart; jj<jend+1; jj++) {
1121: if (jj) {
1122: /* ii and kk must be zero */
1123: cols[cnt++] = slot + gnx*jj;
1124: } else {
1125: /* ii and jj must be zero */
1126: for (kk=kstart; kk<kend+1; kk++) {
1127: cols[cnt++] = slot + gnx*gny*kk;
1128: }
1129: }
1130: }
1131: }
1132: }
1133: }
1134: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1135: }
1136: }
1137: }
1138: PetscFree(values);
1139: PetscFree(cols);
1140: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1141: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1142: return(0);
1143: }
1145: /* BAD! Almost identical to the BAIJ one */
1148: int DAGetMatrix3d_MPISBAIJ(DA da,Mat J)
1149: {
1150: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1151: int m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1152: int istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1153: MPI_Comm comm;
1154: PetscScalar *values;
1155: DAPeriodicType wrap;
1156: DAStencilType st;
1157: ISLocalToGlobalMapping ltog;
1160: /*
1161: nc - number of components per grid point
1162: col - number of colors needed in one direction for single component problem
1163:
1164: */
1165: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1166: if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1167: col = 2*s + 1;
1169: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1170: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1171: PetscObjectGetComm((PetscObject)da,&comm);
1173: /* create the matrix */
1174: PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1175: PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1176: PetscMalloc(col*col*col*sizeof(int),&cols);
1178: DAGetISLocalToGlobalMappingBlck(da,<og);
1180: /* determine the matrix preallocation information */
1181: MatPreallocateSymmetricInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1182: for (i=xs; i<xs+nx; i++) {
1183: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1184: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1185: for (j=ys; j<ys+ny; j++) {
1186: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1187: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1188: for (k=zs; k<zs+nz; k++) {
1189: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1190: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1192: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1194: /* Find block columns in block row */
1195: cnt = 0;
1196: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1197: for (ii=istart; ii<iend+1; ii++) {
1198: for (jj=jstart; jj<jend+1; jj++) {
1199: for (kk=kstart; kk<kend+1; kk++) {
1200: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1201: }
1202: }
1203: }
1204: } else { /* Star stencil */
1205: cnt = 0;
1206: for (ii=istart; ii<iend+1; ii++) {
1207: if (ii) {
1208: /* jj and kk must be zero */
1209: /* cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk; */
1210: cols[cnt++] = slot + ii;
1211: } else {
1212: for (jj=jstart; jj<jend+1; jj++) {
1213: if (jj) {
1214: /* ii and kk must be zero */
1215: cols[cnt++] = slot + gnx*jj;
1216: } else {
1217: /* ii and jj must be zero */
1218: for (kk=kstart; kk<kend+1; kk++) {
1219: cols[cnt++] = slot + gnx*gny*kk;
1220: }
1221: }
1222: }
1223: }
1224: }
1225: }
1226: MatPreallocateSymmetricSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1227: }
1228: }
1229: }
1230: MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1231: MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1232: MatPreallocateFinalize(dnz,onz);
1234: MatSetLocalToGlobalMappingBlock(J,ltog);
1236: /*
1237: For each node in the grid: we get the neighbors in the local (on processor ordering
1238: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1239: PETSc ordering.
1240: */
1242: for (i=xs; i<xs+nx; i++) {
1243: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1244: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1245: for (j=ys; j<ys+ny; j++) {
1246: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1247: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1248: for (k=zs; k<zs+nz; k++) {
1249: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1250: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1251:
1252: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1253:
1254: cnt = 0;
1255: if (st == DA_STENCIL_BOX) { /* if using BOX stencil */
1256: for (ii=istart; ii<iend+1; ii++) {
1257: for (jj=jstart; jj<jend+1; jj++) {
1258: for (kk=kstart; kk<kend+1; kk++) {
1259: cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1260: }
1261: }
1262: }
1263: } else { /* Star stencil */
1264: cnt = 0;
1265: for (ii=istart; ii<iend+1; ii++) {
1266: if (ii) {
1267: /* jj and kk must be zero */
1268: cols[cnt++] = slot + ii;
1269: } else {
1270: for (jj=jstart; jj<jend+1; jj++) {
1271: if (jj) {
1272: /* ii and kk must be zero */
1273: cols[cnt++] = slot + gnx*jj;
1274: } else {
1275: /* ii and jj must be zero */
1276: for (kk=kstart; kk<kend+1; kk++) {
1277: cols[cnt++] = slot + gnx*gny*kk;
1278: }
1279: }
1280: }
1281: }
1282: }
1283: }
1284: MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1285: }
1286: }
1287: }
1288: PetscFree(values);
1289: PetscFree(cols);
1290: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1291: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1292: return(0);
1293: }
1296: /* ---------------------------------------------------------------------------------*/
1300: int DAGetMatrix3d_MPIAIJ_Fill(DA da,Mat J)
1301: {
1302: int ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1303: int m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
1304: int istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1305: int ifill_col,*dfill = da->dfill,*ofill = da->ofill;
1306: int dims[3],starts[3];
1307: MPI_Comm comm;
1308: PetscScalar *values;
1309: DAPeriodicType wrap;
1310: ISLocalToGlobalMapping ltog;
1311: DAStencilType st;
1314: /*
1315: nc - number of components per grid point
1316: col - number of colors needed in one direction for single component problem
1317:
1318: */
1319: DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1320: col = 2*s + 1;
1321: if (DAXPeriodic(wrap) && (m % col)){
1322: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1323: by 2*stencil_width + 1\n");
1324: }
1325: if (DAYPeriodic(wrap) && (n % col)){
1326: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1327: by 2*stencil_width + 1\n");
1328: }
1329: if (DAZPeriodic(wrap) && (p % col)){
1330: SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1331: by 2*stencil_width + 1\n");
1332: }
1334: DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1335: DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1336: PetscObjectGetComm((PetscObject)da,&comm);
1338: PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1339: PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1340: PetscMalloc(nc*sizeof(int),&rows);
1341: PetscMalloc(col*col*col*nc*sizeof(int),&cols);
1342: DAGetISLocalToGlobalMapping(da,<og);
1344: /* determine the matrix preallocation information */
1345: MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1348: for (i=xs; i<xs+nx; i++) {
1349: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1350: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1351: for (j=ys; j<ys+ny; j++) {
1352: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1353: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1354: for (k=zs; k<zs+nz; k++) {
1355: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1356: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1357:
1358: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1359:
1360: for (l=0; l<nc; l++) {
1361: cnt = 0;
1362: for (ii=istart; ii<iend+1; ii++) {
1363: for (jj=jstart; jj<jend+1; jj++) {
1364: for (kk=kstart; kk<kend+1; kk++) {
1365: if (ii || jj || kk) {
1366: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1367: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1368: cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1369: }
1370: } else {
1371: if (dfill) {
1372: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1373: cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1374: } else {
1375: for (ifill_col=0; ifill_col<nc; ifill_col++)
1376: cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1377: }
1378: }
1379: }
1380: }
1381: }
1382: rows[0] = l + nc*(slot);
1383: MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
1384: }
1385: }
1386: }
1387: }
1388: MatSeqAIJSetPreallocation(J,0,dnz);
1389: MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1390: MatPreallocateFinalize(dnz,onz);
1392: MatSetLocalToGlobalMapping(J,ltog);
1393: DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
1394: MatSetStencil(J,3,dims,starts,nc);
1396: /*
1397: For each node in the grid: we get the neighbors in the local (on processor ordering
1398: that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1399: PETSc ordering.
1400: */
1401: for (i=xs; i<xs+nx; i++) {
1402: istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1403: iend = DAXPeriodic(wrap) ? s : (PetscMin(s,m-i-1));
1404: for (j=ys; j<ys+ny; j++) {
1405: jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1406: jend = DAYPeriodic(wrap) ? s : (PetscMin(s,n-j-1));
1407: for (k=zs; k<zs+nz; k++) {
1408: kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1409: kend = DAZPeriodic(wrap) ? s : (PetscMin(s,p-k-1));
1410:
1411: slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1412:
1413: for (l=0; l<nc; l++) {
1414: cnt = 0;
1415: for (ii=istart; ii<iend+1; ii++) {
1416: for (jj=jstart; jj<jend+1; jj++) {
1417: for (kk=kstart; kk<kend+1; kk++) {
1418: if (ii || jj || kk) {
1419: if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1420: for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1421: cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1422: }
1423: } else {
1424: if (dfill) {
1425: for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1426: cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1427: } else {
1428: for (ifill_col=0; ifill_col<nc; ifill_col++)
1429: cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1430: }
1431: }
1432: }
1433: }
1434: }
1435: rows[0] = l + nc*(slot);
1436: MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
1437: }
1438: }
1439: }
1440: }
1441: PetscFree(values);
1442: PetscFree(rows);
1443: PetscFree(cols);
1444: MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1445: MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1446: return(0);
1447: }