Actual source code: fdda.c

petsc-dev 2014-02-02
Report Typos and Errors
  2: #include <petsc-private/dmdaimpl.h> /*I      "petscdmda.h"     I*/
  3: #include <petscmat.h>
  4: #include <petsc-private/matimpl.h>

  6: extern PetscErrorCode DMCreateColoring_DA_1d_MPIAIJ(DM,ISColoringType,ISColoring*);
  7: extern PetscErrorCode DMCreateColoring_DA_2d_MPIAIJ(DM,ISColoringType,ISColoring*);
  8: extern PetscErrorCode DMCreateColoring_DA_2d_5pt_MPIAIJ(DM,ISColoringType,ISColoring*);
  9: extern PetscErrorCode DMCreateColoring_DA_3d_MPIAIJ(DM,ISColoringType,ISColoring*);

 11: /*
 12:    For ghost i that may be negative or greater than the upper bound this
 13:   maps it into the 0:m-1 range using periodicity
 14: */
 15: #define SetInRange(i,m) ((i < 0) ? m+i : ((i >= m) ? i-m : i))

 19: static PetscErrorCode DMDASetBlockFills_Private(const PetscInt *dfill,PetscInt w,PetscInt **rfill)
 20: {
 22:   PetscInt       i,j,nz,*fill;

 25:   if (!dfill) return(0);

 27:   /* count number nonzeros */
 28:   nz = 0;
 29:   for (i=0; i<w; i++) {
 30:     for (j=0; j<w; j++) {
 31:       if (dfill[w*i+j]) nz++;
 32:     }
 33:   }
 34:   PetscMalloc1((nz + w + 1),&fill);
 35:   /* construct modified CSR storage of nonzero structure */
 36:   /*  fill[0 -- w] marks starts of each row of column indices (and end of last row)
 37:    so fill[1] - fill[0] gives number of nonzeros in first row etc */
 38:   nz = w + 1;
 39:   for (i=0; i<w; i++) {
 40:     fill[i] = nz;
 41:     for (j=0; j<w; j++) {
 42:       if (dfill[w*i+j]) {
 43:         fill[nz] = j;
 44:         nz++;
 45:       }
 46:     }
 47:   }
 48:   fill[w] = nz;

 50:   *rfill = fill;
 51:   return(0);
 52: }

 56: /*@
 57:     DMDASetBlockFills - Sets the fill pattern in each block for a multi-component problem
 58:     of the matrix returned by DMCreateMatrix().

 60:     Logically Collective on DMDA

 62:     Input Parameter:
 63: +   da - the distributed array
 64: .   dfill - the fill pattern in the diagonal block (may be NULL, means use dense block)
 65: -   ofill - the fill pattern in the off-diagonal blocks


 68:     Level: developer

 70:     Notes: This only makes sense when you are doing multicomponent problems but using the
 71:        MPIAIJ matrix format

 73:            The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
 74:        representing coupling and 0 entries for missing coupling. For example
 75: $             dfill[9] = {1, 0, 0,
 76: $                         1, 1, 0,
 77: $                         0, 1, 1}
 78:        means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with
 79:        itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the
 80:        diagonal block).

 82:      DMDASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
 83:      can be represented in the dfill, ofill format

 85:    Contributed by Glenn Hammond

 87: .seealso DMCreateMatrix(), DMDASetGetMatrix(), DMSetMatrixPreallocateOnly()

 89: @*/
 90: PetscErrorCode  DMDASetBlockFills(DM da,const PetscInt *dfill,const PetscInt *ofill)
 91: {
 92:   DM_DA          *dd = (DM_DA*)da->data;
 94:   PetscInt       i,k,cnt = 1;

 97:   DMDASetBlockFills_Private(dfill,dd->w,&dd->dfill);
 98:   DMDASetBlockFills_Private(ofill,dd->w,&dd->ofill);

100:   /* ofillcount tracks the columns of ofill that have any nonzero in thems; the value in each location is the number of
101:    columns to the left with any nonzeros in them plus 1 */
102:   PetscCalloc1(dd->w,&dd->ofillcols);
103:   for (i=0; i<dd->w; i++) {
104:     for (k=dd->ofill[i]; k<dd->ofill[i+1]; k++) dd->ofillcols[dd->ofill[k]] = 1;
105:   }
106:   for (i=0; i<dd->w; i++) {
107:     if (dd->ofillcols[i]) {
108:       dd->ofillcols[i] = cnt++;
109:     }
110:   }
111:   return(0);
112: }


117: PetscErrorCode  DMCreateColoring_DA(DM da,ISColoringType ctype,ISColoring *coloring)
118: {
119:   PetscErrorCode   ierr;
120:   PetscInt         dim,m,n,p,nc;
121:   DMDABoundaryType bx,by,bz;
122:   MPI_Comm         comm;
123:   PetscMPIInt      size;
124:   PetscBool        isBAIJ;
125:   DM_DA            *dd = (DM_DA*)da->data;

128:   /*
129:                                   m
130:           ------------------------------------------------------
131:          |                                                     |
132:          |                                                     |
133:          |               ----------------------                |
134:          |               |                    |                |
135:       n  |           yn  |                    |                |
136:          |               |                    |                |
137:          |               .---------------------                |
138:          |             (xs,ys)     xn                          |
139:          |            .                                        |
140:          |         (gxs,gys)                                   |
141:          |                                                     |
142:           -----------------------------------------------------
143:   */

145:   /*
146:          nc - number of components per grid point
147:          col - number of colors needed in one direction for single component problem

149:   */
150:   DMDAGetInfo(da,&dim,0,0,0,&m,&n,&p,&nc,0,&bx,&by,&bz,0);

152:   PetscObjectGetComm((PetscObject)da,&comm);
153:   MPI_Comm_size(comm,&size);
154:   if (ctype == IS_COLORING_GHOSTED) {
155:     if (size == 1) {
156:       ctype = IS_COLORING_GLOBAL;
157:     } else if (dim > 1) {
158:       if ((m==1 && bx == DMDA_BOUNDARY_PERIODIC) || (n==1 && by == DMDA_BOUNDARY_PERIODIC) || (p==1 && bz == DMDA_BOUNDARY_PERIODIC)) {
159:         SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"IS_COLORING_GHOSTED cannot be used for periodic boundary condition having both ends of the domain  on the same process");
160:       }
161:     }
162:   }

164:   /* Tell the DMDA it has 1 degree of freedom per grid point so that the coloring for BAIJ
165:      matrices is for the blocks, not the individual matrix elements  */
166:   PetscStrcmp(da->mattype,MATBAIJ,&isBAIJ);
167:   if (!isBAIJ) {PetscStrcmp(da->mattype,MATMPIBAIJ,&isBAIJ);}
168:   if (!isBAIJ) {PetscStrcmp(da->mattype,MATSEQBAIJ,&isBAIJ);}
169:   if (isBAIJ) {
170:     dd->w  = 1;
171:     dd->xs = dd->xs/nc;
172:     dd->xe = dd->xe/nc;
173:     dd->Xs = dd->Xs/nc;
174:     dd->Xe = dd->Xe/nc;
175:   }

177:   /*
178:      We do not provide a getcoloring function in the DMDA operations because
179:    the basic DMDA does not know about matrices. We think of DMDA as being more
180:    more low-level then matrices.
181:   */
182:   if (dim == 1) {
183:     DMCreateColoring_DA_1d_MPIAIJ(da,ctype,coloring);
184:   } else if (dim == 2) {
185:      DMCreateColoring_DA_2d_MPIAIJ(da,ctype,coloring);
186:   } else if (dim == 3) {
187:      DMCreateColoring_DA_3d_MPIAIJ(da,ctype,coloring);
188:   } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Not done for %D dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
189:   if (isBAIJ) {
190:     dd->w  = nc;
191:     dd->xs = dd->xs*nc;
192:     dd->xe = dd->xe*nc;
193:     dd->Xs = dd->Xs*nc;
194:     dd->Xe = dd->Xe*nc;
195:   }
196:   return(0);
197: }

199: /* ---------------------------------------------------------------------------------*/

203: PetscErrorCode DMCreateColoring_DA_2d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
204: {
205:   PetscErrorCode   ierr;
206:   PetscInt         xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,M,N,dim,s,k,nc,col;
207:   PetscInt         ncolors;
208:   MPI_Comm         comm;
209:   DMDABoundaryType bx,by;
210:   DMDAStencilType  st;
211:   ISColoringValue  *colors;
212:   DM_DA            *dd = (DM_DA*)da->data;

215:   /*
216:          nc - number of components per grid point
217:          col - number of colors needed in one direction for single component problem

219:   */
220:   DMDAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&bx,&by,0,&st);
221:   col  = 2*s + 1;
222:   DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
223:   DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
224:   PetscObjectGetComm((PetscObject)da,&comm);

226:   /* special case as taught to us by Paul Hovland */
227:   if (st == DMDA_STENCIL_STAR && s == 1) {
228:     DMCreateColoring_DA_2d_5pt_MPIAIJ(da,ctype,coloring);
229:   } else {

231:     if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X (%d) is divisible\n\
232:                                                             by 2*stencil_width + 1 (%d)\n", m, col);
233:     if (by == DMDA_BOUNDARY_PERIODIC && (n % col)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y (%d) is divisible\n\
234:                                                             by 2*stencil_width + 1 (%d)\n", n, col);
235:     if (ctype == IS_COLORING_GLOBAL) {
236:       if (!dd->localcoloring) {
237:         PetscMalloc1(nc*nx*ny,&colors);
238:         ii   = 0;
239:         for (j=ys; j<ys+ny; j++) {
240:           for (i=xs; i<xs+nx; i++) {
241:             for (k=0; k<nc; k++) {
242:               colors[ii++] = k + nc*((i % col) + col*(j % col));
243:             }
244:           }
245:         }
246:         ncolors = nc + nc*(col-1 + col*(col-1));
247:         ISColoringCreate(comm,ncolors,nc*nx*ny,colors,&dd->localcoloring);
248:       }
249:       *coloring = dd->localcoloring;
250:     } else if (ctype == IS_COLORING_GHOSTED) {
251:       if (!dd->ghostedcoloring) {
252:         PetscMalloc1(nc*gnx*gny,&colors);
253:         ii   = 0;
254:         for (j=gys; j<gys+gny; j++) {
255:           for (i=gxs; i<gxs+gnx; i++) {
256:             for (k=0; k<nc; k++) {
257:               /* the complicated stuff is to handle periodic boundaries */
258:               colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
259:             }
260:           }
261:         }
262:         ncolors = nc + nc*(col - 1 + col*(col-1));
263:         ISColoringCreate(comm,ncolors,nc*gnx*gny,colors,&dd->ghostedcoloring);
264:         /* PetscIntView(ncolors,(PetscInt*)colors,0); */

266:         ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
267:       }
268:       *coloring = dd->ghostedcoloring;
269:     } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
270:   }
271:   ISColoringReference(*coloring);
272:   return(0);
273: }

275: /* ---------------------------------------------------------------------------------*/

279: PetscErrorCode DMCreateColoring_DA_3d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
280: {
281:   PetscErrorCode   ierr;
282:   PetscInt         xs,ys,nx,ny,i,j,gxs,gys,gnx,gny,m,n,p,dim,s,k,nc,col,zs,gzs,ii,l,nz,gnz,M,N,P;
283:   PetscInt         ncolors;
284:   MPI_Comm         comm;
285:   DMDABoundaryType bx,by,bz;
286:   DMDAStencilType  st;
287:   ISColoringValue  *colors;
288:   DM_DA            *dd = (DM_DA*)da->data;

291:   /*
292:          nc - number of components per grid point
293:          col - number of colors needed in one direction for single component problem

295:   */
296:   DMDAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&bx,&by,&bz,&st);
297:   col  = 2*s + 1;
298:   if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
299:                                                          by 2*stencil_width + 1\n");
300:   if (by == DMDA_BOUNDARY_PERIODIC && (n % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
301:                                                          by 2*stencil_width + 1\n");
302:   if (bz == DMDA_BOUNDARY_PERIODIC && (p % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
303:                                                          by 2*stencil_width + 1\n");

305:   DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
306:   DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
307:   PetscObjectGetComm((PetscObject)da,&comm);

309:   /* create the coloring */
310:   if (ctype == IS_COLORING_GLOBAL) {
311:     if (!dd->localcoloring) {
312:       PetscMalloc1(nc*nx*ny*nz,&colors);
313:       ii   = 0;
314:       for (k=zs; k<zs+nz; k++) {
315:         for (j=ys; j<ys+ny; j++) {
316:           for (i=xs; i<xs+nx; i++) {
317:             for (l=0; l<nc; l++) {
318:               colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
319:             }
320:           }
321:         }
322:       }
323:       ncolors = nc + nc*(col-1 + col*(col-1)+ col*col*(col-1));
324:       ISColoringCreate(comm,ncolors,nc*nx*ny*nz,colors,&dd->localcoloring);
325:     }
326:     *coloring = dd->localcoloring;
327:   } else if (ctype == IS_COLORING_GHOSTED) {
328:     if (!dd->ghostedcoloring) {
329:       PetscMalloc1(nc*gnx*gny*gnz,&colors);
330:       ii   = 0;
331:       for (k=gzs; k<gzs+gnz; k++) {
332:         for (j=gys; j<gys+gny; j++) {
333:           for (i=gxs; i<gxs+gnx; i++) {
334:             for (l=0; l<nc; l++) {
335:               /* the complicated stuff is to handle periodic boundaries */
336:               colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
337:             }
338:           }
339:         }
340:       }
341:       ncolors = nc + nc*(col-1 + col*(col-1)+ col*col*(col-1));
342:       ISColoringCreate(comm,ncolors,nc*gnx*gny*gnz,colors,&dd->ghostedcoloring);
343:       ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
344:     }
345:     *coloring = dd->ghostedcoloring;
346:   } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
347:   ISColoringReference(*coloring);
348:   return(0);
349: }

351: /* ---------------------------------------------------------------------------------*/

355: PetscErrorCode DMCreateColoring_DA_1d_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
356: {
357:   PetscErrorCode   ierr;
358:   PetscInt         xs,nx,i,i1,gxs,gnx,l,m,M,dim,s,nc,col;
359:   PetscInt         ncolors;
360:   MPI_Comm         comm;
361:   DMDABoundaryType bx;
362:   ISColoringValue  *colors;
363:   DM_DA            *dd = (DM_DA*)da->data;

366:   /*
367:          nc - number of components per grid point
368:          col - number of colors needed in one direction for single component problem

370:   */
371:   DMDAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&bx,0,0,0);
372:   col  = 2*s + 1;

374:   if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ2(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points %d is divisible\n\
375:                                                           by 2*stencil_width + 1 %d\n",(int)m,(int)col);

377:   DMDAGetCorners(da,&xs,0,0,&nx,0,0);
378:   DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
379:   PetscObjectGetComm((PetscObject)da,&comm);

381:   /* create the coloring */
382:   if (ctype == IS_COLORING_GLOBAL) {
383:     if (!dd->localcoloring) {
384:       PetscMalloc1(nc*nx,&colors);
385:       if (dd->ofillcols) {
386:         PetscInt tc = 0;
387:         for (i=0; i<nc; i++) tc += (PetscInt) (dd->ofillcols[i] > 0);
388:         i1 = 0;
389:         for (i=xs; i<xs+nx; i++) {
390:           for (l=0; l<nc; l++) {
391:             if (dd->ofillcols[l] && (i % col)) {
392:               colors[i1++] =  nc - 1 + tc*((i % col) - 1) + dd->ofillcols[l];
393:             } else {
394:               colors[i1++] = l;
395:             }
396:           }
397:         }
398:         ncolors = nc + 2*s*tc;
399:       } else {
400:         i1 = 0;
401:         for (i=xs; i<xs+nx; i++) {
402:           for (l=0; l<nc; l++) {
403:             colors[i1++] = l + nc*(i % col);
404:           }
405:         }
406:         ncolors = nc + nc*(col-1);
407:       }
408:       ISColoringCreate(comm,ncolors,nc*nx,colors,&dd->localcoloring);
409:     }
410:     *coloring = dd->localcoloring;
411:   } else if (ctype == IS_COLORING_GHOSTED) {
412:     if (!dd->ghostedcoloring) {
413:       PetscMalloc1(nc*gnx,&colors);
414:       i1   = 0;
415:       for (i=gxs; i<gxs+gnx; i++) {
416:         for (l=0; l<nc; l++) {
417:           /* the complicated stuff is to handle periodic boundaries */
418:           colors[i1++] = l + nc*(SetInRange(i,m) % col);
419:         }
420:       }
421:       ncolors = nc + nc*(col-1);
422:       ISColoringCreate(comm,ncolors,nc*gnx,colors,&dd->ghostedcoloring);
423:       ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
424:     }
425:     *coloring = dd->ghostedcoloring;
426:   } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
427:   ISColoringReference(*coloring);
428:   return(0);
429: }

433: PetscErrorCode DMCreateColoring_DA_2d_5pt_MPIAIJ(DM da,ISColoringType ctype,ISColoring *coloring)
434: {
435:   PetscErrorCode   ierr;
436:   PetscInt         xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,dim,s,k,nc;
437:   PetscInt         ncolors;
438:   MPI_Comm         comm;
439:   DMDABoundaryType bx,by;
440:   ISColoringValue  *colors;
441:   DM_DA            *dd = (DM_DA*)da->data;

444:   /*
445:          nc - number of components per grid point
446:          col - number of colors needed in one direction for single component problem

448:   */
449:   DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,0);
450:   DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
451:   DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
452:   PetscObjectGetComm((PetscObject)da,&comm);

454:   if (bx == DMDA_BOUNDARY_PERIODIC && (m % 5)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible by 5\n");
455:   if (by == DMDA_BOUNDARY_PERIODIC && (n % 5)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible by 5\n");

457:   /* create the coloring */
458:   if (ctype == IS_COLORING_GLOBAL) {
459:     if (!dd->localcoloring) {
460:       PetscMalloc1(nc*nx*ny,&colors);
461:       ii   = 0;
462:       for (j=ys; j<ys+ny; j++) {
463:         for (i=xs; i<xs+nx; i++) {
464:           for (k=0; k<nc; k++) {
465:             colors[ii++] = k + nc*((3*j+i) % 5);
466:           }
467:         }
468:       }
469:       ncolors = 5*nc;
470:       ISColoringCreate(comm,ncolors,nc*nx*ny,colors,&dd->localcoloring);
471:     }
472:     *coloring = dd->localcoloring;
473:   } else if (ctype == IS_COLORING_GHOSTED) {
474:     if (!dd->ghostedcoloring) {
475:       PetscMalloc1(nc*gnx*gny,&colors);
476:       ii = 0;
477:       for (j=gys; j<gys+gny; j++) {
478:         for (i=gxs; i<gxs+gnx; i++) {
479:           for (k=0; k<nc; k++) {
480:             colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
481:           }
482:         }
483:       }
484:       ncolors = 5*nc;
485:       ISColoringCreate(comm,ncolors,nc*gnx*gny,colors,&dd->ghostedcoloring);
486:       ISColoringSetType(dd->ghostedcoloring,IS_COLORING_GHOSTED);
487:     }
488:     *coloring = dd->ghostedcoloring;
489:   } else SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
490:   return(0);
491: }

493: /* =========================================================================== */
494: extern PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ(DM,Mat);
495: extern PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ_Fill(DM,Mat);
496: extern PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ(DM,Mat);
497: extern PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ_Fill(DM,Mat);
498: extern PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ(DM,Mat);
499: extern PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ_Fill(DM,Mat);
500: extern PetscErrorCode DMCreateMatrix_DA_2d_MPIBAIJ(DM,Mat);
501: extern PetscErrorCode DMCreateMatrix_DA_3d_MPIBAIJ(DM,Mat);
502: extern PetscErrorCode DMCreateMatrix_DA_2d_MPISBAIJ(DM,Mat);
503: extern PetscErrorCode DMCreateMatrix_DA_3d_MPISBAIJ(DM,Mat);

507: /*@C
508:    MatSetupDM - Sets the DMDA that is to be used by the HYPRE_StructMatrix PETSc matrix

510:    Logically Collective on Mat

512:    Input Parameters:
513: +  mat - the matrix
514: -  da - the da

516:    Level: intermediate

518: @*/
519: PetscErrorCode MatSetupDM(Mat mat,DM da)
520: {

526:   PetscTryMethod(mat,"MatSetupDM_C",(Mat,DM),(mat,da));
527:   return(0);
528: }

532: PetscErrorCode  MatView_MPI_DA(Mat A,PetscViewer viewer)
533: {
534:   DM                da;
535:   PetscErrorCode    ierr;
536:   const char        *prefix;
537:   Mat               Anatural;
538:   AO                ao;
539:   PetscInt          rstart,rend,*petsc,i;
540:   IS                is;
541:   MPI_Comm          comm;
542:   PetscViewerFormat format;

545:   /* Check whether we are just printing info, in which case MatView() already viewed everything we wanted to view */
546:   PetscViewerGetFormat(viewer,&format);
547:   if (format == PETSC_VIEWER_ASCII_INFO || format == PETSC_VIEWER_ASCII_INFO_DETAIL) return(0);

549:   PetscObjectGetComm((PetscObject)A,&comm);
550:   MatGetDM(A, &da);
551:   if (!da) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONG,"Matrix not generated from a DMDA");

553:   DMDAGetAO(da,&ao);
554:   MatGetOwnershipRange(A,&rstart,&rend);
555:   PetscMalloc1((rend-rstart),&petsc);
556:   for (i=rstart; i<rend; i++) petsc[i-rstart] = i;
557:   AOApplicationToPetsc(ao,rend-rstart,petsc);
558:   ISCreateGeneral(comm,rend-rstart,petsc,PETSC_OWN_POINTER,&is);

560:   /* call viewer on natural ordering */
561:   MatGetSubMatrix(A,is,is,MAT_INITIAL_MATRIX,&Anatural);
562:   ISDestroy(&is);
563:   PetscObjectGetOptionsPrefix((PetscObject)A,&prefix);
564:   PetscObjectSetOptionsPrefix((PetscObject)Anatural,prefix);
565:   PetscObjectSetName((PetscObject)Anatural,((PetscObject)A)->name);
566:   MatView(Anatural,viewer);
567:   MatDestroy(&Anatural);
568:   return(0);
569: }

573: PetscErrorCode  MatLoad_MPI_DA(Mat A,PetscViewer viewer)
574: {
575:   DM             da;
577:   Mat            Anatural,Aapp;
578:   AO             ao;
579:   PetscInt       rstart,rend,*app,i;
580:   IS             is;
581:   MPI_Comm       comm;

584:   PetscObjectGetComm((PetscObject)A,&comm);
585:   MatGetDM(A, &da);
586:   if (!da) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONG,"Matrix not generated from a DMDA");

588:   /* Load the matrix in natural ordering */
589:   MatCreate(PetscObjectComm((PetscObject)A),&Anatural);
590:   MatSetType(Anatural,((PetscObject)A)->type_name);
591:   MatSetSizes(Anatural,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);
592:   MatLoad(Anatural,viewer);

594:   /* Map natural ordering to application ordering and create IS */
595:   DMDAGetAO(da,&ao);
596:   MatGetOwnershipRange(Anatural,&rstart,&rend);
597:   PetscMalloc1((rend-rstart),&app);
598:   for (i=rstart; i<rend; i++) app[i-rstart] = i;
599:   AOPetscToApplication(ao,rend-rstart,app);
600:   ISCreateGeneral(comm,rend-rstart,app,PETSC_OWN_POINTER,&is);

602:   /* Do permutation and replace header */
603:   MatGetSubMatrix(Anatural,is,is,MAT_INITIAL_MATRIX,&Aapp);
604:   MatHeaderReplace(A,Aapp);
605:   ISDestroy(&is);
606:   MatDestroy(&Anatural);
607:   return(0);
608: }

612: PetscErrorCode DMCreateMatrix_DA(DM da, Mat *J)
613: {
615:   PetscInt       dim,dof,nx,ny,nz,dims[3],starts[3],M,N,P;
616:   Mat            A;
617:   MPI_Comm       comm;
618:   MatType        Atype;
619:   PetscSection   section, sectionGlobal;
620:   void           (*aij)(void)=NULL,(*baij)(void)=NULL,(*sbaij)(void)=NULL;
621:   MatType        mtype;
622:   PetscMPIInt    size;
623:   DM_DA          *dd = (DM_DA*)da->data;

626:   MatInitializePackage();
627:   mtype = da->mattype;

629:   DMGetDefaultSection(da, &section);
630:   if (section) {
631:     PetscInt  bs = -1;
632:     PetscInt  localSize;
633:     PetscBool isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isSymmetric;

635:     DMGetDefaultGlobalSection(da, &sectionGlobal);
636:     PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize);
637:     MatCreate(PetscObjectComm((PetscObject)da), J);
638:     MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE);
639:     MatSetType(*J, mtype);
640:     MatSetFromOptions(*J);
641:     PetscStrcmp(mtype, MATSHELL, &isShell);
642:     PetscStrcmp(mtype, MATBAIJ, &isBlock);
643:     PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock);
644:     PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock);
645:     PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);
646:     PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);
647:     PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);
648:     /* Check for symmetric storage */
649:     isSymmetric = (PetscBool) (isSymBlock || isSymSeqBlock || isSymMPIBlock);
650:     if (isSymmetric) {
651:       MatSetOption(*J, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);
652:     }
653:     if (!isShell) {
654:       PetscInt *dnz, *onz, *dnzu, *onzu, bsLocal;

656:       if (bs < 0) {
657:         if (isBlock || isSeqBlock || isMPIBlock || isSymBlock || isSymSeqBlock || isSymMPIBlock) {
658:           PetscInt pStart, pEnd, p, dof;

660:           PetscSectionGetChart(sectionGlobal, &pStart, &pEnd);
661:           for (p = pStart; p < pEnd; ++p) {
662:             PetscSectionGetDof(sectionGlobal, p, &dof);
663:             if (dof) {
664:               bs = dof;
665:               break;
666:             }
667:           }
668:         } else {
669:           bs = 1;
670:         }
671:         /* Must have same blocksize on all procs (some might have no points) */
672:         bsLocal = bs;
673:         MPI_Allreduce(&bsLocal, &bs, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)da));
674:       }
675:       PetscCalloc4(localSize/bs, &dnz, localSize/bs, &onz, localSize/bs, &dnzu, localSize/bs, &onzu);
676:       /* DMPlexPreallocateOperator(dm, bs, section, sectionGlobal, dnz, onz, dnzu, onzu, *J, fillMatrix); */
677:       PetscFree4(dnz, onz, dnzu, onzu);
678:     }
679:   }
680:   /*
681:                                   m
682:           ------------------------------------------------------
683:          |                                                     |
684:          |                                                     |
685:          |               ----------------------                |
686:          |               |                    |                |
687:       n  |           ny  |                    |                |
688:          |               |                    |                |
689:          |               .---------------------                |
690:          |             (xs,ys)     nx                          |
691:          |            .                                        |
692:          |         (gxs,gys)                                   |
693:          |                                                     |
694:           -----------------------------------------------------
695:   */

697:   /*
698:          nc - number of components per grid point
699:          col - number of colors needed in one direction for single component problem

701:   */
702:   M   = dd->M;
703:   N   = dd->N;
704:   P   = dd->P;
705:   dim = dd->dim;
706:   dof = dd->w;
707:   /* DMDAGetInfo(da,&dim,&M,&N,&P,0,0,0,&dof,0,0,0,0,0); */
708:   DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);
709:   PetscObjectGetComm((PetscObject)da,&comm);
710:   MatCreate(comm,&A);
711:   MatSetSizes(A,dof*nx*ny*nz,dof*nx*ny*nz,dof*M*N*P,dof*M*N*P);
712:   MatSetType(A,mtype);
713:   MatSetDM(A,da);
714:   MatSetFromOptions(A);
715:   MatGetType(A,&Atype);
716:   /*
717:      We do not provide a getmatrix function in the DMDA operations because
718:    the basic DMDA does not know about matrices. We think of DMDA as being more
719:    more low-level than matrices. This is kind of cheating but, cause sometimes
720:    we think of DMDA has higher level than matrices.

722:      We could switch based on Atype (or mtype), but we do not since the
723:    specialized setting routines depend only the particular preallocation
724:    details of the matrix, not the type itself.
725:   */
726:   PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
727:   if (!aij) {
728:     PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
729:   }
730:   if (!aij) {
731:     PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
732:     if (!baij) {
733:       PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
734:     }
735:     if (!baij) {
736:       PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
737:       if (!sbaij) {
738:         PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
739:       }
740:     }
741:   }
742:   if (aij) {
743:     if (dim == 1) {
744:       if (dd->ofill) {
745:         DMCreateMatrix_DA_1d_MPIAIJ_Fill(da,A);
746:       } else {
747:         DMCreateMatrix_DA_1d_MPIAIJ(da,A);
748:       }
749:     } else if (dim == 2) {
750:       if (dd->ofill) {
751:         DMCreateMatrix_DA_2d_MPIAIJ_Fill(da,A);
752:       } else {
753:         DMCreateMatrix_DA_2d_MPIAIJ(da,A);
754:       }
755:     } else if (dim == 3) {
756:       if (dd->ofill) {
757:         DMCreateMatrix_DA_3d_MPIAIJ_Fill(da,A);
758:       } else {
759:         DMCreateMatrix_DA_3d_MPIAIJ(da,A);
760:       }
761:     }
762:   } else if (baij) {
763:     if (dim == 2) {
764:       DMCreateMatrix_DA_2d_MPIBAIJ(da,A);
765:     } else if (dim == 3) {
766:       DMCreateMatrix_DA_3d_MPIBAIJ(da,A);
767:     } else SETERRQ3(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s in %D dimension! Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype,dim);
768:   } else if (sbaij) {
769:     if (dim == 2) {
770:       DMCreateMatrix_DA_2d_MPISBAIJ(da,A);
771:     } else if (dim == 3) {
772:       DMCreateMatrix_DA_3d_MPISBAIJ(da,A);
773:     } else SETERRQ3(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s in %D dimension! Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype,dim);
774:   } else {
775:     ISLocalToGlobalMapping ltog,ltogb;
776:     DMGetLocalToGlobalMapping(da,&ltog);
777:     DMGetLocalToGlobalMappingBlock(da,&ltogb);
778:     MatSetUp(A);
779:     MatSetLocalToGlobalMapping(A,ltog,ltog);
780:     MatSetLocalToGlobalMappingBlock(A,ltogb,ltogb);
781:   }
782:   DMDAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
783:   MatSetStencil(A,dim,dims,starts,dof);
784:   MatSetDM(A,da);
785:   MPI_Comm_size(comm,&size);
786:   if (size > 1) {
787:     /* change viewer to display matrix in natural ordering */
788:     MatShellSetOperation(A, MATOP_VIEW, (void (*)(void))MatView_MPI_DA);
789:     MatShellSetOperation(A, MATOP_LOAD, (void (*)(void))MatLoad_MPI_DA);
790:   }
791:   *J = A;
792:   return(0);
793: }

795: /* ---------------------------------------------------------------------------------*/
798: PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ(DM da,Mat J)
799: {
800:   PetscErrorCode         ierr;
801:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny,m,n,dim,s,*cols = NULL,k,nc,*rows = NULL,col,cnt,l,p;
802:   PetscInt               lstart,lend,pstart,pend,*dnz,*onz;
803:   MPI_Comm               comm;
804:   PetscScalar            *values;
805:   DMDABoundaryType       bx,by;
806:   ISLocalToGlobalMapping ltog,ltogb;
807:   DMDAStencilType        st;

810:   /*
811:          nc - number of components per grid point
812:          col - number of colors needed in one direction for single component problem

814:   */
815:   DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
816:   col  = 2*s + 1;
817:   DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
818:   DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
819:   PetscObjectGetComm((PetscObject)da,&comm);

821:   PetscMalloc2(nc,&rows,col*col*nc*nc,&cols);
822:   DMGetLocalToGlobalMapping(da,&ltog);
823:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

825:   /* determine the matrix preallocation information */
826:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
827:   for (i=xs; i<xs+nx; i++) {

829:     pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
830:     pend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));

832:     for (j=ys; j<ys+ny; j++) {
833:       slot = i - gxs + gnx*(j - gys);

835:       lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
836:       lend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));

838:       cnt = 0;
839:       for (k=0; k<nc; k++) {
840:         for (l=lstart; l<lend+1; l++) {
841:           for (p=pstart; p<pend+1; p++) {
842:             if ((st == DMDA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
843:               cols[cnt++] = k + nc*(slot + gnx*l + p);
844:             }
845:           }
846:         }
847:         rows[k] = k + nc*(slot);
848:       }
849:       MatPreallocateSetLocal(ltog,nc,rows,ltog,cnt,cols,dnz,onz);
850:     }
851:   }
852:   MatSetBlockSize(J,nc);
853:   MatSeqAIJSetPreallocation(J,0,dnz);
854:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
855:   MatPreallocateFinalize(dnz,onz);

857:   MatSetLocalToGlobalMapping(J,ltog,ltog);
858:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

860:   /*
861:     For each node in the grid: we get the neighbors in the local (on processor ordering
862:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
863:     PETSc ordering.
864:   */
865:   if (!da->prealloc_only) {
866:     PetscCalloc1(col*col*nc*nc,&values);
867:     for (i=xs; i<xs+nx; i++) {

869:       pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
870:       pend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));

872:       for (j=ys; j<ys+ny; j++) {
873:         slot = i - gxs + gnx*(j - gys);

875:         lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
876:         lend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));

878:         cnt = 0;
879:         for (k=0; k<nc; k++) {
880:           for (l=lstart; l<lend+1; l++) {
881:             for (p=pstart; p<pend+1; p++) {
882:               if ((st == DMDA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
883:                 cols[cnt++] = k + nc*(slot + gnx*l + p);
884:               }
885:             }
886:           }
887:           rows[k] = k + nc*(slot);
888:         }
889:         MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
890:       }
891:     }
892:     PetscFree(values);
893:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
894:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
895:   }
896:   PetscFree2(rows,cols);
897:   return(0);
898: }

902: PetscErrorCode DMCreateMatrix_DA_2d_MPIAIJ_Fill(DM da,Mat J)
903: {
904:   PetscErrorCode         ierr;
905:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
906:   PetscInt               m,n,dim,s,*cols,k,nc,row,col,cnt,l,p;
907:   PetscInt               lstart,lend,pstart,pend,*dnz,*onz;
908:   DM_DA                  *dd = (DM_DA*)da->data;
909:   PetscInt               ifill_col,*ofill = dd->ofill, *dfill = dd->dfill;
910:   MPI_Comm               comm;
911:   PetscScalar            *values;
912:   DMDABoundaryType       bx,by;
913:   ISLocalToGlobalMapping ltog,ltogb;
914:   DMDAStencilType        st;

917:   /*
918:          nc - number of components per grid point
919:          col - number of colors needed in one direction for single component problem

921:   */
922:   DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
923:   col  = 2*s + 1;
924:   DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
925:   DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
926:   PetscObjectGetComm((PetscObject)da,&comm);

928:   PetscMalloc1(col*col*nc*nc,&cols);
929:   DMGetLocalToGlobalMapping(da,&ltog);
930:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

932:   /* determine the matrix preallocation information */
933:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
934:   for (i=xs; i<xs+nx; i++) {

936:     pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
937:     pend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));

939:     for (j=ys; j<ys+ny; j++) {
940:       slot = i - gxs + gnx*(j - gys);

942:       lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
943:       lend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));

945:       for (k=0; k<nc; k++) {
946:         cnt = 0;
947:         for (l=lstart; l<lend+1; l++) {
948:           for (p=pstart; p<pend+1; p++) {
949:             if (l || p) {
950:               if ((st == DMDA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
951:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
952:               }
953:             } else {
954:               if (dfill) {
955:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
956:               } else {
957:                 for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
958:               }
959:             }
960:           }
961:         }
962:         row  = k + nc*(slot);
963:         MatPreallocateSetLocal(ltog,1,&row,ltog,cnt,cols,dnz,onz);
964:       }
965:     }
966:   }
967:   MatSeqAIJSetPreallocation(J,0,dnz);
968:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
969:   MatPreallocateFinalize(dnz,onz);
970:   MatSetLocalToGlobalMapping(J,ltog,ltog);
971:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

973:   /*
974:     For each node in the grid: we get the neighbors in the local (on processor ordering
975:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
976:     PETSc ordering.
977:   */
978:   if (!da->prealloc_only) {
979:     PetscCalloc1(col*col*nc*nc,&values);
980:     for (i=xs; i<xs+nx; i++) {

982:       pstart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
983:       pend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));

985:       for (j=ys; j<ys+ny; j++) {
986:         slot = i - gxs + gnx*(j - gys);

988:         lstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
989:         lend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));

991:         for (k=0; k<nc; k++) {
992:           cnt = 0;
993:           for (l=lstart; l<lend+1; l++) {
994:             for (p=pstart; p<pend+1; p++) {
995:               if (l || p) {
996:                 if ((st == DMDA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
997:                   for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + gnx*l + p);
998:                 }
999:               } else {
1000:                 if (dfill) {
1001:                   for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + gnx*l + p);
1002:                 } else {
1003:                   for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + gnx*l + p);
1004:                 }
1005:               }
1006:             }
1007:           }
1008:           row  = k + nc*(slot);
1009:           MatSetValuesLocal(J,1,&row,cnt,cols,values,INSERT_VALUES);
1010:         }
1011:       }
1012:     }
1013:     PetscFree(values);
1014:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1015:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1016:   }
1017:   PetscFree(cols);
1018:   return(0);
1019: }

1021: /* ---------------------------------------------------------------------------------*/

1025: PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ(DM da,Mat J)
1026: {
1027:   PetscErrorCode         ierr;
1028:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1029:   PetscInt               m,n,dim,s,*cols = NULL,k,nc,*rows = NULL,col,cnt,l,p,*dnz = NULL,*onz = NULL;
1030:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1031:   MPI_Comm               comm;
1032:   PetscScalar            *values;
1033:   DMDABoundaryType       bx,by,bz;
1034:   ISLocalToGlobalMapping ltog,ltogb;
1035:   DMDAStencilType        st;

1038:   /*
1039:          nc - number of components per grid point
1040:          col - number of colors needed in one direction for single component problem

1042:   */
1043:   DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1044:   col  = 2*s + 1;

1046:   DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1047:   DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1048:   PetscObjectGetComm((PetscObject)da,&comm);

1050:   PetscMalloc2(nc,&rows,col*col*col*nc*nc,&cols);
1051:   DMGetLocalToGlobalMapping(da,&ltog);
1052:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

1054:   /* determine the matrix preallocation information */
1055:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
1056:   for (i=xs; i<xs+nx; i++) {
1057:     istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1058:     iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1059:     for (j=ys; j<ys+ny; j++) {
1060:       jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1061:       jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1062:       for (k=zs; k<zs+nz; k++) {
1063:         kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1064:         kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1066:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1068:         cnt = 0;
1069:         for (l=0; l<nc; l++) {
1070:           for (ii=istart; ii<iend+1; ii++) {
1071:             for (jj=jstart; jj<jend+1; jj++) {
1072:               for (kk=kstart; kk<kend+1; kk++) {
1073:                 if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1074:                   cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1075:                 }
1076:               }
1077:             }
1078:           }
1079:           rows[l] = l + nc*(slot);
1080:         }
1081:         MatPreallocateSetLocal(ltog,nc,rows,ltog,cnt,cols,dnz,onz);
1082:       }
1083:     }
1084:   }
1085:   MatSetBlockSize(J,nc);
1086:   MatSeqAIJSetPreallocation(J,0,dnz);
1087:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1088:   MatPreallocateFinalize(dnz,onz);
1089:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1090:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1092:   /*
1093:     For each node in the grid: we get the neighbors in the local (on processor ordering
1094:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1095:     PETSc ordering.
1096:   */
1097:   if (!da->prealloc_only) {
1098:     PetscCalloc1(col*col*col*nc*nc*nc,&values);
1099:     for (i=xs; i<xs+nx; i++) {
1100:       istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1101:       iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1102:       for (j=ys; j<ys+ny; j++) {
1103:         jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1104:         jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1105:         for (k=zs; k<zs+nz; k++) {
1106:           kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1107:           kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1109:           slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1111:           cnt = 0;
1112:           for (l=0; l<nc; l++) {
1113:             for (ii=istart; ii<iend+1; ii++) {
1114:               for (jj=jstart; jj<jend+1; jj++) {
1115:                 for (kk=kstart; kk<kend+1; kk++) {
1116:                   if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1117:                     cols[cnt++] = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1118:                   }
1119:                 }
1120:               }
1121:             }
1122:             rows[l] = l + nc*(slot);
1123:           }
1124:           MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
1125:         }
1126:       }
1127:     }
1128:     PetscFree(values);
1129:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1130:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1131:   }
1132:   PetscFree2(rows,cols);
1133:   return(0);
1134: }

1136: /* ---------------------------------------------------------------------------------*/

1140: PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ_Fill(DM da,Mat J)
1141: {
1142:   PetscErrorCode         ierr;
1143:   DM_DA                  *dd = (DM_DA*)da->data;
1144:   PetscInt               xs,nx,i,j,gxs,gnx,row,k,l;
1145:   PetscInt               m,dim,s,*cols = NULL,nc,col,cnt,*ocols;
1146:   PetscInt               *ofill = dd->ofill,*dfill = dd->dfill;
1147:   PetscScalar            *values;
1148:   DMDABoundaryType       bx;
1149:   ISLocalToGlobalMapping ltog,ltogb;
1150:   PetscMPIInt            rank,size;

1153:   if (dd->bx == DMDA_BOUNDARY_PERIODIC) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"With fill provided not implemented with periodic boundary conditions");
1154:   MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);
1155:   MPI_Comm_size(PetscObjectComm((PetscObject)da),&size);

1157:   /*
1158:          nc - number of components per grid point
1159:          col - number of colors needed in one direction for single component problem

1161:   */
1162:   DMDAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&bx,0,0,0);
1163:   col  = 2*s + 1;

1165:   DMDAGetCorners(da,&xs,0,0,&nx,0,0);
1166:   DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);

1168:   MatSetBlockSize(J,nc);
1169:   PetscCalloc2(nx*nc,&cols,nx*nc,&ocols);

1171:   if (nx < 2) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Need at least two grid points per process");
1172:   /*
1173:         note should be smaller for first and last process with no periodic
1174:         does not handle dfill
1175:   */
1176:   cnt = 0;
1177:   /* coupling with process to the left */
1178:   for (i=0; i<s; i++) {
1179:     for (j=0; j<nc; j++) {
1180:       ocols[cnt] = ((!rank) ? 0 : (s - i)*(ofill[j+1] - ofill[j]));
1181:       cols[cnt]  = dfill[j+1] - dfill[j] + (s + i)*(ofill[j+1] - ofill[j]);
1182:       cnt++;
1183:     }
1184:   }
1185:   for (i=s; i<nx-s; i++) {
1186:     for (j=0; j<nc; j++) {
1187:       cols[cnt] = dfill[j+1] - dfill[j] + 2*s*(ofill[j+1] - ofill[j]);
1188:       cnt++;
1189:     }
1190:   }
1191:   /* coupling with process to the right */
1192:   for (i=nx-s; i<nx; i++) {
1193:     for (j=0; j<nc; j++) {
1194:       ocols[cnt] = ((rank == (size-1)) ? 0 : (i - nx + s + 1)*(ofill[j+1] - ofill[j]));
1195:       cols[cnt]  = dfill[j+1] - dfill[j] + (s + nx - i - 1)*(ofill[j+1] - ofill[j]);
1196:       cnt++;
1197:     }
1198:   }

1200:   MatSeqAIJSetPreallocation(J,0,cols);
1201:   MatMPIAIJSetPreallocation(J,0,cols,0,ocols);
1202:   PetscFree2(cols,ocols);

1204:   DMGetLocalToGlobalMapping(da,&ltog);
1205:   DMGetLocalToGlobalMappingBlock(da,&ltogb);
1206:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1207:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1209:   /*
1210:     For each node in the grid: we get the neighbors in the local (on processor ordering
1211:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1212:     PETSc ordering.
1213:   */
1214:   if (!da->prealloc_only) {
1215:     PetscMalloc1(col*nc*nc,&cols);
1216:     PetscCalloc1(col*nc*nc,&values);

1218:     row = xs*nc;
1219:     /* coupling with process to the left */
1220:     for (i=xs; i<xs+s; i++) {
1221:       for (j=0; j<nc; j++) {
1222:         cnt = 0;
1223:         if (rank) {
1224:           for (l=0; l<s; l++) {
1225:             for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i - s + l)*nc + ofill[k];
1226:           }
1227:         }
1228:         if (dfill) {
1229:           for (k=dfill[j]; k<dfill[j+1]; k++) {
1230:             cols[cnt++] = i*nc + dfill[k];
1231:           }
1232:         } else {
1233:           for (k=0; k<nc; k++) {
1234:             cols[cnt++] = i*nc + k;
1235:           }
1236:         }
1237:         for (l=0; l<s; l++) {
1238:           for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i + s - l)*nc + ofill[k];
1239:         }
1240:         MatSetValues(J,1,&row,cnt,cols,values,INSERT_VALUES);
1241:         row++;
1242:       }
1243:     }
1244:     for (i=xs+s; i<xs+nx-s; i++) {
1245:       for (j=0; j<nc; j++) {
1246:         cnt = 0;
1247:         for (l=0; l<s; l++) {
1248:           for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i - s + l)*nc + ofill[k];
1249:         }
1250:         if (dfill) {
1251:           for (k=dfill[j]; k<dfill[j+1]; k++) {
1252:             cols[cnt++] = i*nc + dfill[k];
1253:           }
1254:         } else {
1255:           for (k=0; k<nc; k++) {
1256:             cols[cnt++] = i*nc + k;
1257:           }
1258:         }
1259:         for (l=0; l<s; l++) {
1260:           for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i + s - l)*nc + ofill[k];
1261:         }
1262:         MatSetValues(J,1,&row,cnt,cols,values,INSERT_VALUES);
1263:         row++;
1264:       }
1265:     }
1266:     /* coupling with process to the right */
1267:     for (i=xs+nx-s; i<xs+nx; i++) {
1268:       for (j=0; j<nc; j++) {
1269:         cnt = 0;
1270:         for (l=0; l<s; l++) {
1271:           for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i - s + l)*nc + ofill[k];
1272:         }
1273:         if (dfill) {
1274:           for (k=dfill[j]; k<dfill[j+1]; k++) {
1275:             cols[cnt++] = i*nc + dfill[k];
1276:           }
1277:         } else {
1278:           for (k=0; k<nc; k++) {
1279:             cols[cnt++] = i*nc + k;
1280:           }
1281:         }
1282:         if (rank < size-1) {
1283:           for (l=0; l<s; l++) {
1284:             for (k=ofill[j]; k<ofill[j+1]; k++) cols[cnt++] = (i + s - l)*nc + ofill[k];
1285:           }
1286:         }
1287:         MatSetValues(J,1,&row,cnt,cols,values,INSERT_VALUES);
1288:         row++;
1289:       }
1290:     }
1291:     PetscFree(values);
1292:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1293:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1294:     PetscFree(cols);
1295:   }
1296:   return(0);
1297: }

1299: /* ---------------------------------------------------------------------------------*/

1303: PetscErrorCode DMCreateMatrix_DA_1d_MPIAIJ(DM da,Mat J)
1304: {
1305:   PetscErrorCode         ierr;
1306:   PetscInt               xs,nx,i,i1,slot,gxs,gnx;
1307:   PetscInt               m,dim,s,*cols = NULL,nc,*rows = NULL,col,cnt,l;
1308:   PetscInt               istart,iend;
1309:   PetscScalar            *values;
1310:   DMDABoundaryType       bx;
1311:   ISLocalToGlobalMapping ltog,ltogb;

1314:   /*
1315:          nc - number of components per grid point
1316:          col - number of colors needed in one direction for single component problem

1318:   */
1319:   DMDAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&bx,0,0,0);
1320:   col  = 2*s + 1;

1322:   DMDAGetCorners(da,&xs,0,0,&nx,0,0);
1323:   DMDAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);

1325:   MatSetBlockSize(J,nc);
1326:   MatSeqAIJSetPreallocation(J,col*nc,0);
1327:   MatMPIAIJSetPreallocation(J,col*nc,0,col*nc,0);

1329:   DMGetLocalToGlobalMapping(da,&ltog);
1330:   DMGetLocalToGlobalMappingBlock(da,&ltogb);
1331:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1332:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1334:   /*
1335:     For each node in the grid: we get the neighbors in the local (on processor ordering
1336:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1337:     PETSc ordering.
1338:   */
1339:   if (!da->prealloc_only) {
1340:     PetscMalloc2(nc,&rows,col*nc*nc,&cols);
1341:     PetscCalloc1(col*nc*nc,&values);
1342:     for (i=xs; i<xs+nx; i++) {
1343:       istart = PetscMax(-s,gxs - i);
1344:       iend   = PetscMin(s,gxs + gnx - i - 1);
1345:       slot   = i - gxs;

1347:       cnt = 0;
1348:       for (l=0; l<nc; l++) {
1349:         for (i1=istart; i1<iend+1; i1++) {
1350:           cols[cnt++] = l + nc*(slot + i1);
1351:         }
1352:         rows[l] = l + nc*(slot);
1353:       }
1354:       MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
1355:     }
1356:     PetscFree(values);
1357:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1358:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1359:     PetscFree2(rows,cols);
1360:   }
1361:   return(0);
1362: }

1366: PetscErrorCode DMCreateMatrix_DA_2d_MPIBAIJ(DM da,Mat J)
1367: {
1368:   PetscErrorCode         ierr;
1369:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1370:   PetscInt               m,n,dim,s,*cols,nc,col,cnt,*dnz,*onz;
1371:   PetscInt               istart,iend,jstart,jend,ii,jj;
1372:   MPI_Comm               comm;
1373:   PetscScalar            *values;
1374:   DMDABoundaryType       bx,by;
1375:   DMDAStencilType        st;
1376:   ISLocalToGlobalMapping ltog,ltogb;

1379:   /*
1380:      nc - number of components per grid point
1381:      col - number of colors needed in one direction for single component problem
1382:   */
1383:   DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
1384:   col  = 2*s + 1;

1386:   DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
1387:   DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
1388:   PetscObjectGetComm((PetscObject)da,&comm);

1390:   PetscMalloc1(col*col*nc*nc,&cols);

1392:   DMGetLocalToGlobalMapping(da,&ltog);
1393:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

1395:   /* determine the matrix preallocation information */
1396:   MatPreallocateInitialize(comm,nx*ny,nx*ny,dnz,onz);
1397:   for (i=xs; i<xs+nx; i++) {
1398:     istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1399:     iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1400:     for (j=ys; j<ys+ny; j++) {
1401:       jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1402:       jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1403:       slot   = i - gxs + gnx*(j - gys);

1405:       /* Find block columns in block row */
1406:       cnt = 0;
1407:       for (ii=istart; ii<iend+1; ii++) {
1408:         for (jj=jstart; jj<jend+1; jj++) {
1409:           if (st == DMDA_STENCIL_BOX || !ii || !jj) { /* BOX or on the STAR */
1410:             cols[cnt++] = slot + ii + gnx*jj;
1411:           }
1412:         }
1413:       }
1414:       MatPreallocateSetLocal(ltogb,1,&slot,ltogb,cnt,cols,dnz,onz);
1415:     }
1416:   }
1417:   MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1418:   MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1419:   MatPreallocateFinalize(dnz,onz);

1421:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1422:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1424:   /*
1425:     For each node in the grid: we get the neighbors in the local (on processor ordering
1426:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1427:     PETSc ordering.
1428:   */
1429:   if (!da->prealloc_only) {
1430:     PetscCalloc1(col*col*nc*nc,&values);
1431:     for (i=xs; i<xs+nx; i++) {
1432:       istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1433:       iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1434:       for (j=ys; j<ys+ny; j++) {
1435:         jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1436:         jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1437:         slot = i - gxs + gnx*(j - gys);
1438:         cnt  = 0;
1439:         for (ii=istart; ii<iend+1; ii++) {
1440:           for (jj=jstart; jj<jend+1; jj++) {
1441:             if (st == DMDA_STENCIL_BOX || !ii || !jj) { /* BOX or on the STAR */
1442:               cols[cnt++] = slot + ii + gnx*jj;
1443:             }
1444:           }
1445:         }
1446:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1447:       }
1448:     }
1449:     PetscFree(values);
1450:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1451:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1452:   }
1453:   PetscFree(cols);
1454:   return(0);
1455: }

1459: PetscErrorCode DMCreateMatrix_DA_3d_MPIBAIJ(DM da,Mat J)
1460: {
1461:   PetscErrorCode         ierr;
1462:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1463:   PetscInt               m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1464:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1465:   MPI_Comm               comm;
1466:   PetscScalar            *values;
1467:   DMDABoundaryType       bx,by,bz;
1468:   DMDAStencilType        st;
1469:   ISLocalToGlobalMapping ltog,ltogb;

1472:   /*
1473:          nc - number of components per grid point
1474:          col - number of colors needed in one direction for single component problem

1476:   */
1477:   DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1478:   col  = 2*s + 1;

1480:   DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1481:   DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1482:   PetscObjectGetComm((PetscObject)da,&comm);

1484:   PetscMalloc1(col*col*col,&cols);

1486:   DMGetLocalToGlobalMapping(da,&ltog);
1487:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

1489:   /* determine the matrix preallocation information */
1490:   MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1491:   for (i=xs; i<xs+nx; i++) {
1492:     istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1493:     iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1494:     for (j=ys; j<ys+ny; j++) {
1495:       jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1496:       jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1497:       for (k=zs; k<zs+nz; k++) {
1498:         kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1499:         kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1501:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1503:         /* Find block columns in block row */
1504:         cnt = 0;
1505:         for (ii=istart; ii<iend+1; ii++) {
1506:           for (jj=jstart; jj<jend+1; jj++) {
1507:             for (kk=kstart; kk<kend+1; kk++) {
1508:               if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1509:                 cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1510:               }
1511:             }
1512:           }
1513:         }
1514:         MatPreallocateSetLocal(ltogb,1,&slot,ltogb,cnt,cols,dnz,onz);
1515:       }
1516:     }
1517:   }
1518:   MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1519:   MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1520:   MatPreallocateFinalize(dnz,onz);

1522:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1523:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1525:   /*
1526:     For each node in the grid: we get the neighbors in the local (on processor ordering
1527:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1528:     PETSc ordering.
1529:   */
1530:   if (!da->prealloc_only) {
1531:     PetscCalloc1(col*col*col*nc*nc,&values);
1532:     for (i=xs; i<xs+nx; i++) {
1533:       istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1534:       iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1535:       for (j=ys; j<ys+ny; j++) {
1536:         jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1537:         jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1538:         for (k=zs; k<zs+nz; k++) {
1539:           kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1540:           kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1542:           slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1544:           cnt = 0;
1545:           for (ii=istart; ii<iend+1; ii++) {
1546:             for (jj=jstart; jj<jend+1; jj++) {
1547:               for (kk=kstart; kk<kend+1; kk++) {
1548:                 if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1549:                   cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1550:                 }
1551:               }
1552:             }
1553:           }
1554:           MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1555:         }
1556:       }
1557:     }
1558:     PetscFree(values);
1559:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1560:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1561:   }
1562:   PetscFree(cols);
1563:   return(0);
1564: }

1568: /*
1569:   This helper is for of SBAIJ preallocation, to discard the lower-triangular values which are difficult to
1570:   identify in the local ordering with periodic domain.
1571: */
1572: static PetscErrorCode L2GFilterUpperTriangular(ISLocalToGlobalMapping ltog,PetscInt *row,PetscInt *cnt,PetscInt col[])
1573: {
1575:   PetscInt       i,n;

1578:   ISLocalToGlobalMappingApply(ltog,1,row,row);
1579:   ISLocalToGlobalMappingApply(ltog,*cnt,col,col);
1580:   for (i=0,n=0; i<*cnt; i++) {
1581:     if (col[i] >= *row) col[n++] = col[i];
1582:   }
1583:   *cnt = n;
1584:   return(0);
1585: }

1589: PetscErrorCode DMCreateMatrix_DA_2d_MPISBAIJ(DM da,Mat J)
1590: {
1591:   PetscErrorCode         ierr;
1592:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1593:   PetscInt               m,n,dim,s,*cols,nc,col,cnt,*dnz,*onz;
1594:   PetscInt               istart,iend,jstart,jend,ii,jj;
1595:   MPI_Comm               comm;
1596:   PetscScalar            *values;
1597:   DMDABoundaryType       bx,by;
1598:   DMDAStencilType        st;
1599:   ISLocalToGlobalMapping ltog,ltogb;

1602:   /*
1603:      nc - number of components per grid point
1604:      col - number of colors needed in one direction for single component problem
1605:   */
1606:   DMDAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&bx,&by,0,&st);
1607:   col  = 2*s + 1;

1609:   DMDAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
1610:   DMDAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
1611:   PetscObjectGetComm((PetscObject)da,&comm);

1613:   PetscMalloc1(col*col*nc*nc,&cols);

1615:   DMGetLocalToGlobalMapping(da,&ltog);
1616:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

1618:   /* determine the matrix preallocation information */
1619:   MatPreallocateInitialize(comm,nx*ny,nx*ny,dnz,onz);
1620:   for (i=xs; i<xs+nx; i++) {
1621:     istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1622:     iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1623:     for (j=ys; j<ys+ny; j++) {
1624:       jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1625:       jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1626:       slot   = i - gxs + gnx*(j - gys);

1628:       /* Find block columns in block row */
1629:       cnt = 0;
1630:       for (ii=istart; ii<iend+1; ii++) {
1631:         for (jj=jstart; jj<jend+1; jj++) {
1632:           if (st == DMDA_STENCIL_BOX || !ii || !jj) {
1633:             cols[cnt++] = slot + ii + gnx*jj;
1634:           }
1635:         }
1636:       }
1637:       L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1638:       MatPreallocateSymmetricSet(slot,cnt,cols,dnz,onz);
1639:     }
1640:   }
1641:   MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1642:   MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1643:   MatPreallocateFinalize(dnz,onz);

1645:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1646:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1648:   /*
1649:     For each node in the grid: we get the neighbors in the local (on processor ordering
1650:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1651:     PETSc ordering.
1652:   */
1653:   if (!da->prealloc_only) {
1654:     PetscCalloc1(col*col*nc*nc,&values);
1655:     for (i=xs; i<xs+nx; i++) {
1656:       istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1657:       iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1658:       for (j=ys; j<ys+ny; j++) {
1659:         jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1660:         jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1661:         slot   = i - gxs + gnx*(j - gys);

1663:         /* Find block columns in block row */
1664:         cnt = 0;
1665:         for (ii=istart; ii<iend+1; ii++) {
1666:           for (jj=jstart; jj<jend+1; jj++) {
1667:             if (st == DMDA_STENCIL_BOX || !ii || !jj) {
1668:               cols[cnt++] = slot + ii + gnx*jj;
1669:             }
1670:           }
1671:         }
1672:         L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1673:         MatSetValuesBlocked(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1674:       }
1675:     }
1676:     PetscFree(values);
1677:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1678:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1679:   }
1680:   PetscFree(cols);
1681:   return(0);
1682: }

1686: PetscErrorCode DMCreateMatrix_DA_3d_MPISBAIJ(DM da,Mat J)
1687: {
1688:   PetscErrorCode         ierr;
1689:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1690:   PetscInt               m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1691:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1692:   MPI_Comm               comm;
1693:   PetscScalar            *values;
1694:   DMDABoundaryType       bx,by,bz;
1695:   DMDAStencilType        st;
1696:   ISLocalToGlobalMapping ltog,ltogb;

1699:   /*
1700:      nc - number of components per grid point
1701:      col - number of colors needed in one direction for single component problem
1702:   */
1703:   DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1704:   col  = 2*s + 1;

1706:   DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1707:   DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1708:   PetscObjectGetComm((PetscObject)da,&comm);

1710:   /* create the matrix */
1711:   PetscMalloc1(col*col*col,&cols);

1713:   DMGetLocalToGlobalMapping(da,&ltog);
1714:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

1716:   /* determine the matrix preallocation information */
1717:   MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1718:   for (i=xs; i<xs+nx; i++) {
1719:     istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1720:     iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1721:     for (j=ys; j<ys+ny; j++) {
1722:       jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1723:       jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1724:       for (k=zs; k<zs+nz; k++) {
1725:         kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1726:         kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1728:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1730:         /* Find block columns in block row */
1731:         cnt = 0;
1732:         for (ii=istart; ii<iend+1; ii++) {
1733:           for (jj=jstart; jj<jend+1; jj++) {
1734:             for (kk=kstart; kk<kend+1; kk++) {
1735:               if ((st == DMDA_STENCIL_BOX) || (!ii && !jj) || (!jj && !kk) || (!ii && !kk)) {
1736:                 cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1737:               }
1738:             }
1739:           }
1740:         }
1741:         L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1742:         MatPreallocateSymmetricSet(slot,cnt,cols,dnz,onz);
1743:       }
1744:     }
1745:   }
1746:   MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1747:   MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1748:   MatPreallocateFinalize(dnz,onz);

1750:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1751:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1753:   /*
1754:     For each node in the grid: we get the neighbors in the local (on processor ordering
1755:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1756:     PETSc ordering.
1757:   */
1758:   if (!da->prealloc_only) {
1759:     PetscCalloc1(col*col*col*nc*nc,&values);
1760:     for (i=xs; i<xs+nx; i++) {
1761:       istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1762:       iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1763:       for (j=ys; j<ys+ny; j++) {
1764:         jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1765:         jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1766:         for (k=zs; k<zs+nz; k++) {
1767:           kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1768:           kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1770:           slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1772:           cnt = 0;
1773:           for (ii=istart; ii<iend+1; ii++) {
1774:             for (jj=jstart; jj<jend+1; jj++) {
1775:               for (kk=kstart; kk<kend+1; kk++) {
1776:                 if ((st == DMDA_STENCIL_BOX) || (!ii && !jj) || (!jj && !kk) || (!ii && !kk)) {
1777:                   cols[cnt++] = slot + ii + gnx*jj + gnx*gny*kk;
1778:                 }
1779:               }
1780:             }
1781:           }
1782:           L2GFilterUpperTriangular(ltogb,&slot,&cnt,cols);
1783:           MatSetValuesBlocked(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1784:         }
1785:       }
1786:     }
1787:     PetscFree(values);
1788:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1789:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1790:   }
1791:   PetscFree(cols);
1792:   return(0);
1793: }

1795: /* ---------------------------------------------------------------------------------*/

1799: PetscErrorCode DMCreateMatrix_DA_3d_MPIAIJ_Fill(DM da,Mat J)
1800: {
1801:   PetscErrorCode         ierr;
1802:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1803:   PetscInt               m,n,dim,s,*cols,k,nc,row,col,cnt,l,p,*dnz,*onz;
1804:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1805:   DM_DA                  *dd = (DM_DA*)da->data;
1806:   PetscInt               ifill_col,*dfill = dd->dfill,*ofill = dd->ofill;
1807:   MPI_Comm               comm;
1808:   PetscScalar            *values;
1809:   DMDABoundaryType       bx,by,bz;
1810:   ISLocalToGlobalMapping ltog,ltogb;
1811:   DMDAStencilType        st;

1814:   /*
1815:          nc - number of components per grid point
1816:          col - number of colors needed in one direction for single component problem

1818:   */
1819:   DMDAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&bx,&by,&bz,&st);
1820:   col  = 2*s + 1;
1821:   if (bx == DMDA_BOUNDARY_PERIODIC && (m % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1822:                  by 2*stencil_width + 1\n");
1823:   if (by == DMDA_BOUNDARY_PERIODIC && (n % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1824:                  by 2*stencil_width + 1\n");
1825:   if (bz == DMDA_BOUNDARY_PERIODIC && (p % col)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1826:                  by 2*stencil_width + 1\n");

1828:   DMDAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1829:   DMDAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1830:   PetscObjectGetComm((PetscObject)da,&comm);

1832:   PetscMalloc1(col*col*col*nc,&cols);
1833:   DMGetLocalToGlobalMapping(da,&ltog);
1834:   DMGetLocalToGlobalMappingBlock(da,&ltogb);

1836:   /* determine the matrix preallocation information */
1837:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);


1840:   for (i=xs; i<xs+nx; i++) {
1841:     istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1842:     iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1843:     for (j=ys; j<ys+ny; j++) {
1844:       jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1845:       jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1846:       for (k=zs; k<zs+nz; k++) {
1847:         kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1848:         kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1850:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1852:         for (l=0; l<nc; l++) {
1853:           cnt = 0;
1854:           for (ii=istart; ii<iend+1; ii++) {
1855:             for (jj=jstart; jj<jend+1; jj++) {
1856:               for (kk=kstart; kk<kend+1; kk++) {
1857:                 if (ii || jj || kk) {
1858:                   if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1859:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1860:                   }
1861:                 } else {
1862:                   if (dfill) {
1863:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1864:                   } else {
1865:                     for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1866:                   }
1867:                 }
1868:               }
1869:             }
1870:           }
1871:           row  = l + nc*(slot);
1872:           MatPreallocateSetLocal(ltog,1,&row,ltog,cnt,cols,dnz,onz);
1873:         }
1874:       }
1875:     }
1876:   }
1877:   MatSeqAIJSetPreallocation(J,0,dnz);
1878:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1879:   MatPreallocateFinalize(dnz,onz);
1880:   MatSetLocalToGlobalMapping(J,ltog,ltog);
1881:   MatSetLocalToGlobalMappingBlock(J,ltogb,ltogb);

1883:   /*
1884:     For each node in the grid: we get the neighbors in the local (on processor ordering
1885:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1886:     PETSc ordering.
1887:   */
1888:   if (!da->prealloc_only) {
1889:     PetscCalloc1(col*col*col*nc*nc*nc,&values);
1890:     for (i=xs; i<xs+nx; i++) {
1891:       istart = (bx == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-i));
1892:       iend   = (bx == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,m-i-1));
1893:       for (j=ys; j<ys+ny; j++) {
1894:         jstart = (by == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-j));
1895:         jend   = (by == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,n-j-1));
1896:         for (k=zs; k<zs+nz; k++) {
1897:           kstart = (bz == DMDA_BOUNDARY_PERIODIC) ? -s : (PetscMax(-s,-k));
1898:           kend   = (bz == DMDA_BOUNDARY_PERIODIC) ?  s : (PetscMin(s,p-k-1));

1900:           slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1902:           for (l=0; l<nc; l++) {
1903:             cnt = 0;
1904:             for (ii=istart; ii<iend+1; ii++) {
1905:               for (jj=jstart; jj<jend+1; jj++) {
1906:                 for (kk=kstart; kk<kend+1; kk++) {
1907:                   if (ii || jj || kk) {
1908:                     if ((st == DMDA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1909:                       for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++) cols[cnt++] = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1910:                     }
1911:                   } else {
1912:                     if (dfill) {
1913:                       for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++) cols[cnt++] = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1914:                     } else {
1915:                       for (ifill_col=0; ifill_col<nc; ifill_col++) cols[cnt++] = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1916:                     }
1917:                   }
1918:                 }
1919:               }
1920:             }
1921:             row  = l + nc*(slot);
1922:             MatSetValuesLocal(J,1,&row,cnt,cols,values,INSERT_VALUES);
1923:           }
1924:         }
1925:       }
1926:     }
1927:     PetscFree(values);
1928:     MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1929:     MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1930:   }
1931:   PetscFree(cols);
1932:   return(0);
1933: }