Actual source code: fdda.c

  1: 
 2:  #include src/dm/da/daimpl.h
 3:  #include petscmat.h


  6: EXTERN PetscErrorCode DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
  7: EXTERN PetscErrorCode DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
  8: EXTERN PetscErrorCode DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
  9: EXTERN PetscErrorCode DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);

 11: /*
 12:    For ghost i that may be negative or greater than the upper bound this
 13:   maps it into the 0:m-1 range using periodicity
 14: */
 15: #define SetInRange(i,m) ((i < 0) ? m+i:((i >= m) ? i-m:i))

 19: static PetscErrorCode DASetBlockFills_Private(PetscInt *dfill,PetscInt w,PetscInt **rfill)
 20: {
 22:   PetscInt       i,j,nz,*fill;

 25:   /* count number nonzeros */
 26:   nz = 0;
 27:   for (i=0; i<w; i++) {
 28:     for (j=0; j<w; j++) {
 29:       if (dfill[w*i+j]) nz++;
 30:     }
 31:   }
 32:   PetscMalloc((nz + w + 1)*sizeof(PetscInt),&fill);
 33:   /* construct modified CSR storage of nonzero structure */
 34:   nz = w + 1;
 35:   for (i=0; i<w; i++) {
 36:     fill[i] = nz;
 37:     for (j=0; j<w; j++) {
 38:       if (dfill[w*i+j]) {
 39:         fill[nz] = j;
 40:         nz++;
 41:       }
 42:     }
 43:   }
 44:   fill[w] = nz;
 45: 
 46:   *rfill = fill;
 47:   return(0);
 48: }

 52: /*@C
 53:     DASetBlockFills - Sets the fill pattern in each block for a multi-component problem
 54:     of the matrix returned by DAGetMatrix().

 56:     Collective on DA

 58:     Input Parameter:
 59: +   da - the distributed array
 60: .   dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
 61: -   ofill - the fill pattern in the off-diagonal blocks


 64:     Level: developer

 66:     Notes: This only makes sense when you are doing multicomponent problems but using the
 67:        MPIAIJ matrix format

 69:            The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
 70:        representing coupling and 0 entries for missing coupling. For example 
 71: $             dfill[3][3] = {1, 0, 0,
 72: $                            1, 1, 0,
 73: $                            0, 1, 1} 
 74:        means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with 
 75:        itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the 
 76:        diagonal block.

 78:      DASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
 79:      can be represented in the dfill, ofill format

 81:    Contributed by Glenn Hammond

 83: .seealso DAGetMatrix(), DASetGetMatrix()

 85: @*/
 86: PetscErrorCode DASetBlockFills(DA da,PetscInt *dfill,PetscInt *ofill)
 87: {

 91:   if (dfill) {
 92:     DASetBlockFills_Private(dfill,da->w,&da->dfill);
 93:   }
 94:   DASetBlockFills_Private(ofill,da->w,&da->ofill);
 95:   return(0);
 96: }


101: /*@C
102:     DAGetColoring - Gets the coloring required for computing the Jacobian via
103:     finite differences on a function defined using a stencil on the DA.

105:     Collective on DA

107:     Input Parameter:
108: +   da - the distributed array
109: -   ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED

111:     Output Parameters:
112: .   coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)

114:     Level: advanced

116:     Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used 
117:    for efficient (parallel or thread based) triangular solves etc is NOT yet 
118:    available. 


121: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring

123: @*/
124: PetscErrorCode DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
125: {
127:   PetscInt       dim;

130:   /*
131:                                   m
132:           ------------------------------------------------------
133:          |                                                     |
134:          |                                                     |
135:          |               ----------------------                |
136:          |               |                    |                |
137:       n  |           yn  |                    |                |
138:          |               |                    |                |
139:          |               .---------------------                |
140:          |             (xs,ys)     xn                          |
141:          |            .                                        |
142:          |         (gxs,gys)                                   |
143:          |                                                     |
144:           -----------------------------------------------------
145:   */

147:   /*     
148:          nc - number of components per grid point 
149:          col - number of colors needed in one direction for single component problem
150:   
151:   */
152:   DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);

154:   /*
155:      We do not provide a getcoloring function in the DA operations because 
156:    the basic DA does not know about matrices. We think of DA as being more 
157:    more low-level then matrices.
158:   */
159:   if (dim == 1) {
160:     DAGetColoring1d_MPIAIJ(da,ctype,coloring);
161:   } else if (dim == 2) {
162:      DAGetColoring2d_MPIAIJ(da,ctype,coloring);
163:   } else if (dim == 3) {
164:      DAGetColoring3d_MPIAIJ(da,ctype,coloring);
165:   } else {
166:       SETERRQ1(PETSC_ERR_SUP,"Not done for %D dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
167:   }
168:   return(0);
169: }

171: /* ---------------------------------------------------------------------------------*/

175: PetscErrorCode DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
176: {
177:   PetscErrorCode         ierr;
178:   PetscInt               xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,M,N,dim,s,k,nc,col;
179:   PetscMPIInt            size;
180:   MPI_Comm               comm;
181:   DAPeriodicType         wrap;
182:   DAStencilType          st;
183:   ISColoringValue        *colors;

186:   /*     
187:          nc - number of components per grid point 
188:          col - number of colors needed in one direction for single component problem
189:   
190:   */
191:   DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&wrap,&st);
192:   col    = 2*s + 1;
193:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
194:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
195:   PetscObjectGetComm((PetscObject)da,&comm);
196:   MPI_Comm_size(comm,&size);

198:   /* special case as taught to us by Paul Hovland */
199:   if (st == DA_STENCIL_STAR && s == 1) {
200:     DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
201:   } else {

203:     if (DAXPeriodic(wrap) && (m % col)){
204:       SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
205:                  by 2*stencil_width + 1\n");
206:     }
207:     if (DAYPeriodic(wrap) && (n % col)){
208:       SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
209:                  by 2*stencil_width + 1\n");
210:     }
211:     if (ctype == IS_COLORING_LOCAL) {
212:       if (!da->localcoloring) {
213:         PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
214:         ii = 0;
215:         for (j=ys; j<ys+ny; j++) {
216:           for (i=xs; i<xs+nx; i++) {
217:             for (k=0; k<nc; k++) {
218:               colors[ii++] = k + nc*((i % col) + col*(j % col));
219:             }
220:           }
221:         }
222:         ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
223:       }
224:       *coloring = da->localcoloring;
225:     } else if (ctype == IS_COLORING_GHOSTED) {
226:       if (!da->ghostedcoloring) {
227:         PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
228:         ii = 0;
229:         for (j=gys; j<gys+gny; j++) {
230:           for (i=gxs; i<gxs+gnx; i++) {
231:             for (k=0; k<nc; k++) {
232:               /* the complicated stuff is to handle periodic boundaries */
233:               colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
234:             }
235:           }
236:         }
237:         ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
238:         ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
239:       }
240:       *coloring = da->ghostedcoloring;
241:     } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
242:   }
243:   ISColoringReference(*coloring);
244:   return(0);
245: }

247: /* ---------------------------------------------------------------------------------*/

251: PetscErrorCode DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
252: {
253:   PetscErrorCode  ierr;
254:   PetscInt        xs,ys,nx,ny,i,j,gxs,gys,gnx,gny,m,n,p,dim,s,k,nc,col,zs,gzs,ii,l,nz,gnz,M,N,P;
255:   PetscMPIInt     size;
256:   MPI_Comm        comm;
257:   DAPeriodicType  wrap;
258:   DAStencilType   st;
259:   ISColoringValue *colors;

262:   /*     
263:          nc - number of components per grid point 
264:          col - number of colors needed in one direction for single component problem
265:   
266:   */
267:   DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
268:   col    = 2*s + 1;
269:   if (DAXPeriodic(wrap) && (m % col)){
270:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
271:                  by 2*stencil_width + 1\n");
272:   }
273:   if (DAYPeriodic(wrap) && (n % col)){
274:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
275:                  by 2*stencil_width + 1\n");
276:   }
277:   if (DAZPeriodic(wrap) && (p % col)){
278:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
279:                  by 2*stencil_width + 1\n");
280:   }

282:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
283:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
284:   PetscObjectGetComm((PetscObject)da,&comm);
285:   MPI_Comm_size(comm,&size);

287:   /* create the coloring */
288:   if (ctype == IS_COLORING_LOCAL) {
289:     if (!da->localcoloring) {
290:       PetscMalloc(nc*nx*ny*nz*sizeof(ISColoringValue),&colors);
291:       ii = 0;
292:       for (k=zs; k<zs+nz; k++) {
293:         for (j=ys; j<ys+ny; j++) {
294:           for (i=xs; i<xs+nx; i++) {
295:             for (l=0; l<nc; l++) {
296:               colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
297:             }
298:           }
299:         }
300:       }
301:       ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
302:     }
303:     *coloring = da->localcoloring;
304:   } else if (ctype == IS_COLORING_GHOSTED) {
305:     if (!da->ghostedcoloring) {
306:       PetscMalloc(nc*gnx*gny*gnz*sizeof(ISColoringValue),&colors);
307:       ii = 0;
308:       for (k=gzs; k<gzs+gnz; k++) {
309:         for (j=gys; j<gys+gny; j++) {
310:           for (i=gxs; i<gxs+gnx; i++) {
311:             for (l=0; l<nc; l++) {
312:               /* the complicated stuff is to handle periodic boundaries */
313:               colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
314:             }
315:           }
316:         }
317:       }
318:       ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
319:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
320:     }
321:     *coloring = da->ghostedcoloring;
322:   } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
323:   ISColoringReference(*coloring);
324:   return(0);
325: }

327: /* ---------------------------------------------------------------------------------*/

331: PetscErrorCode DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
332: {
333:   PetscErrorCode  ierr;
334:   PetscInt        xs,nx,i,i1,gxs,gnx,l,m,M,dim,s,nc,col;
335:   PetscMPIInt     size;
336:   MPI_Comm        comm;
337:   DAPeriodicType  wrap;
338:   ISColoringValue *colors;

341:   /*     
342:          nc - number of components per grid point 
343:          col - number of colors needed in one direction for single component problem
344:   
345:   */
346:   DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
347:   col    = 2*s + 1;

349:   if (DAXPeriodic(wrap) && (m % col)) {
350:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisible\n\
351:                  by 2*stencil_width + 1\n");
352:   }

354:   DAGetCorners(da,&xs,0,0,&nx,0,0);
355:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
356:   PetscObjectGetComm((PetscObject)da,&comm);
357:   MPI_Comm_size(comm,&size);

359:   /* create the coloring */
360:   if (ctype == IS_COLORING_LOCAL) {
361:     if (!da->localcoloring) {
362:       PetscMalloc(nc*nx*sizeof(ISColoringValue),&colors);
363:       i1 = 0;
364:       for (i=xs; i<xs+nx; i++) {
365:         for (l=0; l<nc; l++) {
366:           colors[i1++] = l + nc*(i % col);
367:         }
368:       }
369:       ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
370:     }
371:     *coloring = da->localcoloring;
372:   } else if (ctype == IS_COLORING_GHOSTED) {
373:     if (!da->ghostedcoloring) {
374:       PetscMalloc(nc*gnx*sizeof(ISColoringValue),&colors);
375:       i1 = 0;
376:       for (i=gxs; i<gxs+gnx; i++) {
377:         for (l=0; l<nc; l++) {
378:           /* the complicated stuff is to handle periodic boundaries */
379:           colors[i1++] = l + nc*(SetInRange(i,m) % col);
380:         }
381:       }
382:       ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
383:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
384:     }
385:     *coloring = da->ghostedcoloring;
386:   } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
387:   ISColoringReference(*coloring);
388:   return(0);
389: }

393: PetscErrorCode DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
394: {
395:   PetscErrorCode  ierr;
396:   PetscInt        xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny,m,n,dim,s,k,nc;
397:   MPI_Comm        comm;
398:   DAPeriodicType  wrap;
399:   ISColoringValue *colors;

402:   /*     
403:          nc - number of components per grid point 
404:          col - number of colors needed in one direction for single component problem
405:   
406:   */
407:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,0);
408:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
409:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
410:   PetscObjectGetComm((PetscObject)da,&comm);

412:   if (DAXPeriodic(wrap) && (m % 5)){
413:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
414:                  by 5\n");
415:   }
416:   if (DAYPeriodic(wrap) && (n % 5)){
417:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
418:                  by 5\n");
419:   }

421:   /* create the coloring */
422:   if (ctype == IS_COLORING_LOCAL) {
423:     if (!da->localcoloring) {
424:       PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
425:       ii = 0;
426:       for (j=ys; j<ys+ny; j++) {
427:         for (i=xs; i<xs+nx; i++) {
428:           for (k=0; k<nc; k++) {
429:             colors[ii++] = k + nc*((3*j+i) % 5);
430:           }
431:         }
432:       }
433:       ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
434:     }
435:     *coloring = da->localcoloring;
436:   } else if (ctype == IS_COLORING_GHOSTED) {
437:     if (!da->ghostedcoloring) {
438:       PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
439:       ii = 0;
440:       for (j=gys; j<gys+gny; j++) {
441:         for (i=gxs; i<gxs+gnx; i++) {
442:           for (k=0; k<nc; k++) {
443:             colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
444:           }
445:         }
446:       }
447:       ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
448:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
449:     }
450:     *coloring = da->ghostedcoloring;
451:   } else SETERRQ1(PETSC_ERR_ARG_WRONG,"Unknown ISColoringType %d",(int)ctype);
452:   return(0);
453: }

455: /* =========================================================================== */
456: EXTERN PetscErrorCode DAGetMatrix1d_MPIAIJ(DA,Mat);
457: EXTERN PetscErrorCode DAGetMatrix2d_MPIAIJ(DA,Mat);
458: EXTERN PetscErrorCode DAGetMatrix2d_MPIAIJ_Fill(DA,Mat);
459: EXTERN PetscErrorCode DAGetMatrix3d_MPIAIJ(DA,Mat);
460: EXTERN PetscErrorCode DAGetMatrix3d_MPIAIJ_Fill(DA,Mat);
461: EXTERN PetscErrorCode DAGetMatrix3d_MPIBAIJ(DA,Mat);
462: EXTERN PetscErrorCode DAGetMatrix3d_MPISBAIJ(DA,Mat);

466: /*@C
467:     DAGetMatrix - Creates a matrix with the correct parallel layout and nonzero structure required for 
468:       computing the Jacobian on a function defined using the stencil set in the DA.

470:     Collective on DA

472:     Input Parameter:
473: +   da - the distributed array
474: -   mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, or
475:             any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).

477:     Output Parameters:
478: .   J  - matrix with the correct nonzero structure
479:         (obviously without the correct Jacobian values)

481:     Level: advanced

483:     Notes: This properly preallocates the number of nonzeros in the sparse matrix so you 
484:        do not need to do it yourself.

486: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()

488: @*/
489: PetscErrorCode DAGetMatrix(DA da,const MatType mtype,Mat *J)
490: {
492:   PetscInt       dim,dof,nx,ny,nz,dims[3],starts[3];
493:   Mat            A;
494:   MPI_Comm       comm;
495:   MatType        Atype;
496:   void           (*aij)(void)=PETSC_NULL,(*baij)(void)=PETSC_NULL,(*sbaij)(void)=PETSC_NULL;

499:   /*
500:                                   m
501:           ------------------------------------------------------
502:          |                                                     |
503:          |                                                     |
504:          |               ----------------------                |
505:          |               |                    |                |
506:       n  |           yn  |                    |                |
507:          |               |                    |                |
508:          |               .---------------------                |
509:          |             (xs,ys)     xn                          |
510:          |            .                                        |
511:          |         (gxs,gys)                                   |
512:          |                                                     |
513:           -----------------------------------------------------
514:   */

516:   /*     
517:          nc - number of components per grid point 
518:          col - number of colors needed in one direction for single component problem
519:   
520:   */
521:   DAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0);
522:   DAGetCorners(da,0,0,0,&nx,&ny,&nz);
523:   PetscObjectGetComm((PetscObject)da,&comm);
524:   MatCreate(comm,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,&A);
525:   MatSetType(A,mtype);
526:   MatSetFromOptions(A);
527:   MatGetType(A,&Atype);
528:   /*
529:      We do not provide a getmatrix function in the DA operations because 
530:    the basic DA does not know about matrices. We think of DA as being more 
531:    more low-level then matrices. This is kind of cheating but, cause sometimes 
532:    we think of DA has higher level then matrices.

534:      We could switch based on Atype (or mtype), but we do not since the
535:    specialized setting routines depend only the particular preallocation
536:    details of the matrix, not the type itself.
537:   */
538:   PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
539:   if (!aij) {
540:     PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
541:   }
542:   if (aij) {
543:     if (dim == 1) {
544:       DAGetMatrix1d_MPIAIJ(da,A);
545:     } else if (dim == 2) {
546:       if (da->ofill) {
547:         DAGetMatrix2d_MPIAIJ_Fill(da,A);
548:       } else {
549:         DAGetMatrix2d_MPIAIJ(da,A);
550:       }
551:     } else if (dim == 3) {
552:       if (da->ofill) {
553:         DAGetMatrix3d_MPIAIJ_Fill(da,A);
554:       } else {
555:         DAGetMatrix3d_MPIAIJ(da,A);
556:       }
557:     }
558:   } else if (dim == 3) {
559:     PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
560:     if (!baij) {
561:       PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
562:     }
563:     if (baij) {
564:       DAGetMatrix3d_MPIBAIJ(da,A);
565:     } else {
566:       PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
567:       if (!sbaij) {
568:         PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
569:       }
570:       if (sbaij) {
571:         DAGetMatrix3d_MPISBAIJ(da,A);
572:       } else {
573:         SETERRQ2(PETSC_ERR_SUP,"Not implemented for %D dimension and Matrix Type: %s!\n" \
574:                                "Send mail to petsc-maint@mcs.anl.gov for code",
575:                                dim,Atype);
576:       }
577:     }
578:   } else {
579:     SETERRQ2(PETSC_ERR_SUP,"Not implemented for %D dimension and matrix type: %s!\n" \
580:                            "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype);
581:   }
582:   DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
583:   MatSetStencil(A,dim,dims,starts,dof);
584:   *J = A;
585:   return(0);
586: }

588: /* ---------------------------------------------------------------------------------*/
591: PetscErrorCode DAGetMatrix2d_MPIAIJ(DA da,Mat J)
592: {
593:   PetscErrorCode         ierr;
594:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny,m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
595:   PetscInt               lstart,lend,pstart,pend,*dnz,*onz;
596:   MPI_Comm               comm;
597:   PetscScalar            *values;
598:   DAPeriodicType         wrap;
599:   ISLocalToGlobalMapping ltog;
600:   DAStencilType          st;

603:   /*     
604:          nc - number of components per grid point 
605:          col - number of colors needed in one direction for single component problem
606:   
607:   */
608:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
609:   col = 2*s + 1;
610:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
611:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
612:   PetscObjectGetComm((PetscObject)da,&comm);

614:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
615:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
616:   PetscMalloc(nc*sizeof(PetscInt),&rows);
617:   PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
618:   DAGetISLocalToGlobalMapping(da,&ltog);
619: 
620:   /* determine the matrix preallocation information */
621:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
622:   for (i=xs; i<xs+nx; i++) {

624:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
625:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

627:     for (j=ys; j<ys+ny; j++) {
628:       slot = i - gxs + gnx*(j - gys);

630:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
631:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

633:       cnt  = 0;
634:       for (k=0; k<nc; k++) {
635:         for (l=lstart; l<lend+1; l++) {
636:           for (p=pstart; p<pend+1; p++) {
637:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
638:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
639:             }
640:           }
641:         }
642:         rows[k] = k + nc*(slot);
643:       }
644:       MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
645:     }
646:   }
647:   MatSeqAIJSetPreallocation(J,0,dnz);
648:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
649:   MatSetBlockSize(J,nc);
650:   MatPreallocateFinalize(dnz,onz);

652:   MatSetLocalToGlobalMapping(J,ltog);

654:   /*
655:     For each node in the grid: we get the neighbors in the local (on processor ordering
656:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
657:     PETSc ordering.
658:   */
659:   for (i=xs; i<xs+nx; i++) {
660: 
661:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
662:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
663: 
664:     for (j=ys; j<ys+ny; j++) {
665:       slot = i - gxs + gnx*(j - gys);
666: 
667:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
668:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

670:       cnt  = 0;
671:       for (k=0; k<nc; k++) {
672:         for (l=lstart; l<lend+1; l++) {
673:           for (p=pstart; p<pend+1; p++) {
674:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
675:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
676:             }
677:           }
678:         }
679:         rows[k]      = k + nc*(slot);
680:       }
681:       MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
682:     }
683:   }
684:   PetscFree(values);
685:   PetscFree(rows);
686:   PetscFree(cols);
687:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
688:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
689:   return(0);
690: }

694: PetscErrorCode DAGetMatrix2d_MPIAIJ_Fill(DA da,Mat J)
695: {
696:   PetscErrorCode         ierr;
697:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
698:   PetscInt               m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
699:   PetscInt               lstart,lend,pstart,pend,*dnz,*onz;
700:   PetscInt               ifill_col,*ofill = da->ofill, *dfill = da->dfill;
701:   MPI_Comm               comm;
702:   PetscScalar            *values;
703:   DAPeriodicType         wrap;
704:   ISLocalToGlobalMapping ltog;
705:   DAStencilType          st;

708:   /*     
709:          nc - number of components per grid point 
710:          col - number of colors needed in one direction for single component problem
711:   
712:   */
713:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
714:   col = 2*s + 1;
715:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
716:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
717:   PetscObjectGetComm((PetscObject)da,&comm);

719:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
720:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
721:   PetscMalloc(nc*sizeof(PetscInt),&rows);
722:   PetscMalloc(col*col*nc*nc*sizeof(PetscInt),&cols);
723:   DAGetISLocalToGlobalMapping(da,&ltog);
724: 
725:   /* determine the matrix preallocation information */
726:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
727:   for (i=xs; i<xs+nx; i++) {

729:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
730:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

732:     for (j=ys; j<ys+ny; j++) {
733:       slot = i - gxs + gnx*(j - gys);

735:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
736:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

738:       for (k=0; k<nc; k++) {
739:         cnt  = 0;
740:         for (l=lstart; l<lend+1; l++) {
741:           for (p=pstart; p<pend+1; p++) {
742:             if (l || p) {
743:               if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
744:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
745:                   cols[cnt++]  = ofill[ifill_col] + nc*(slot + gnx*l + p);
746:               }
747:             } else {
748:               if (dfill) {
749:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
750:                   cols[cnt++]  = dfill[ifill_col] + nc*(slot + gnx*l + p);
751:               } else {
752:                 for (ifill_col=0; ifill_col<nc; ifill_col++)
753:                   cols[cnt++]  = ifill_col + nc*(slot + gnx*l + p);
754:               }
755:             }
756:           }
757:         }
758:         rows[0] = k + nc*(slot);
759:         MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
760:       }
761:     }
762:   }
763:   MatSeqAIJSetPreallocation(J,0,dnz);
764:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
765:   MatPreallocateFinalize(dnz,onz);
766:   MatSetBlockSize(J,nc);
767:   MatSetLocalToGlobalMapping(J,ltog);

769:   /*
770:     For each node in the grid: we get the neighbors in the local (on processor ordering
771:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
772:     PETSc ordering.
773:   */
774:   for (i=xs; i<xs+nx; i++) {
775: 
776:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
777:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
778: 
779:     for (j=ys; j<ys+ny; j++) {
780:       slot = i - gxs + gnx*(j - gys);
781: 
782:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
783:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

785:       for (k=0; k<nc; k++) {
786:         cnt  = 0;
787:         for (l=lstart; l<lend+1; l++) {
788:           for (p=pstart; p<pend+1; p++) {
789:             if (l || p) {
790:               if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
791:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
792:                   cols[cnt++]  = ofill[ifill_col] + nc*(slot + gnx*l + p);
793:               }
794:             } else {
795:               if (dfill) {
796:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
797:                   cols[cnt++]  = dfill[ifill_col] + nc*(slot + gnx*l + p);
798:               } else {
799:                 for (ifill_col=0; ifill_col<nc; ifill_col++)
800:                   cols[cnt++]  = ifill_col + nc*(slot + gnx*l + p);
801:               }
802:             }
803:           }
804:         }
805:         rows[0]      = k + nc*(slot);
806:         MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
807:       }
808:     }
809:   }
810:   PetscFree(values);
811:   PetscFree(rows);
812:   PetscFree(cols);
813:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
814:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
815:   return(0);
816: }

818: /* ---------------------------------------------------------------------------------*/

822: PetscErrorCode DAGetMatrix3d_MPIAIJ(DA da,Mat J)
823: {
824:   PetscErrorCode         ierr;
825:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
826:   PetscInt               m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
827:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
828:   MPI_Comm               comm;
829:   PetscScalar            *values;
830:   DAPeriodicType         wrap;
831:   ISLocalToGlobalMapping ltog;
832:   DAStencilType          st;

835:   /*     
836:          nc - number of components per grid point 
837:          col - number of colors needed in one direction for single component problem
838:   
839:   */
840:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
841:   col    = 2*s + 1;

843:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
844:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
845:   PetscObjectGetComm((PetscObject)da,&comm);

847:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
848:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
849:   PetscMalloc(nc*sizeof(PetscInt),&rows);
850:   PetscMalloc(col*col*col*nc*sizeof(PetscInt),&cols);
851:   DAGetISLocalToGlobalMapping(da,&ltog);

853:   /* determine the matrix preallocation information */
854:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
855:   for (i=xs; i<xs+nx; i++) {
856:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
857:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
858:     for (j=ys; j<ys+ny; j++) {
859:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
860:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
861:       for (k=zs; k<zs+nz; k++) {
862:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
863:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
864: 
865:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
866: 
867:         cnt  = 0;
868:         for (l=0; l<nc; l++) {
869:           for (ii=istart; ii<iend+1; ii++) {
870:             for (jj=jstart; jj<jend+1; jj++) {
871:               for (kk=kstart; kk<kend+1; kk++) {
872:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
873:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
874:                 }
875:               }
876:             }
877:           }
878:           rows[l] = l + nc*(slot);
879:         }
880:         MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
881:       }
882:     }
883:   }
884:   MatSeqAIJSetPreallocation(J,0,dnz);
885:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
886:   MatPreallocateFinalize(dnz,onz);
887:   MatSetBlockSize(J,nc);CHKERRQ(ierr)
888:   MatSetLocalToGlobalMapping(J,ltog);

890:   /*
891:     For each node in the grid: we get the neighbors in the local (on processor ordering
892:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
893:     PETSc ordering.
894:   */
895:   for (i=xs; i<xs+nx; i++) {
896:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
897:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
898:     for (j=ys; j<ys+ny; j++) {
899:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
900:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
901:       for (k=zs; k<zs+nz; k++) {
902:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
903:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
904: 
905:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
906: 
907:         cnt  = 0;
908:         for (l=0; l<nc; l++) {
909:           for (ii=istart; ii<iend+1; ii++) {
910:             for (jj=jstart; jj<jend+1; jj++) {
911:               for (kk=kstart; kk<kend+1; kk++) {
912:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
913:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
914:                 }
915:               }
916:             }
917:           }
918:           rows[l]      = l + nc*(slot);
919:         }
920:         MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
921:       }
922:     }
923:   }
924:   PetscFree(values);
925:   PetscFree(rows);
926:   PetscFree(cols);
927:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
928:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
929:   return(0);
930: }

932: /* ---------------------------------------------------------------------------------*/

936: PetscErrorCode DAGetMatrix1d_MPIAIJ(DA da,Mat J)
937: {
938:   PetscErrorCode         ierr;
939:   PetscInt               xs,nx,i,i1,slot,gxs,gnx;
940:   PetscInt               m,dim,s,*cols,nc,*rows,col,cnt,l;
941:   PetscInt               istart,iend;
942:   PetscScalar            *values;
943:   DAPeriodicType         wrap;
944:   ISLocalToGlobalMapping ltog;

947:   /*     
948:          nc - number of components per grid point 
949:          col - number of colors needed in one direction for single component problem
950:   
951:   */
952:   DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
953:   col    = 2*s + 1;

955:   DAGetCorners(da,&xs,0,0,&nx,0,0);
956:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);

958:   MatSeqAIJSetPreallocation(J,col*nc,0);
959:   MatMPIAIJSetPreallocation(J,col*nc,0,0,0);
960:   MatSetBlockSize(J,nc);
961:   PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
962:   PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
963:   PetscMalloc(nc*sizeof(PetscInt),&rows);
964:   PetscMalloc(col*nc*sizeof(PetscInt),&cols);
965: 
966:   DAGetISLocalToGlobalMapping(da,&ltog);
967:   MatSetLocalToGlobalMapping(J,ltog);
968: 
969:   /*
970:     For each node in the grid: we get the neighbors in the local (on processor ordering
971:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
972:     PETSc ordering.
973:   */
974:   for (i=xs; i<xs+nx; i++) {
975:     istart = PetscMax(-s,gxs - i);
976:     iend   = PetscMin(s,gxs + gnx - i - 1);
977:     slot   = i - gxs;
978: 
979:     cnt  = 0;
980:     for (l=0; l<nc; l++) {
981:       for (i1=istart; i1<iend+1; i1++) {
982:         cols[cnt++] = l + nc*(slot + i1);
983:       }
984:       rows[l]      = l + nc*(slot);
985:     }
986:     MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
987:   }
988:   PetscFree(values);
989:   PetscFree(rows);
990:   PetscFree(cols);
991:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
992:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
993:   return(0);
994: }

998: PetscErrorCode DAGetMatrix3d_MPIBAIJ(DA da,Mat J)
999: {
1000:   PetscErrorCode         ierr;
1001:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1002:   PetscInt               m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1003:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1004:   MPI_Comm               comm;
1005:   PetscScalar            *values;
1006:   DAPeriodicType         wrap;
1007:   DAStencilType          st;
1008:   ISLocalToGlobalMapping ltog;

1011:   /*     
1012:          nc - number of components per grid point 
1013:          col - number of colors needed in one direction for single component problem
1014:   
1015:   */
1016:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1017:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1018:   col    = 2*s + 1;

1020:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1021:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1022:   PetscObjectGetComm((PetscObject)da,&comm);

1024:   PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1025:   PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1026:   PetscMalloc(col*col*col*sizeof(PetscInt),&cols);

1028:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

1030:   /* determine the matrix preallocation information */
1031:   MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1032:   for (i=xs; i<xs+nx; i++) {
1033:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1034:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1035:     for (j=ys; j<ys+ny; j++) {
1036:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1037:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1038:       for (k=zs; k<zs+nz; k++) {
1039:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1040:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

1042:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1044:         /* Find block columns in block row */
1045:         cnt  = 0;
1046:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1047:           for (ii=istart; ii<iend+1; ii++) {
1048:             for (jj=jstart; jj<jend+1; jj++) {
1049:               for (kk=kstart; kk<kend+1; kk++) {
1050:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1051:               }
1052:             }
1053:           }
1054:         } else {  /* Star stencil */
1055:           cnt  = 0;
1056:           for (ii=istart; ii<iend+1; ii++) {
1057:             if (ii) {
1058:               /* jj and kk must be zero */
1059:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
1060:               cols[cnt++]  = slot + ii;
1061:             } else {
1062:               for (jj=jstart; jj<jend+1; jj++) {
1063:                 if (jj) {
1064:                   /* ii and kk must be zero */
1065:                   cols[cnt++]  = slot + gnx*jj;
1066:                 } else {
1067:                   /* ii and jj must be zero */
1068:                   for (kk=kstart; kk<kend+1; kk++) {
1069:                     cols[cnt++]  = slot + gnx*gny*kk;
1070:                   }
1071:                 }
1072:               }
1073:             }
1074:           }
1075:         }
1076:         MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1077:       }
1078:     }
1079:   }
1080:   MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1081:   MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1082:   MatPreallocateFinalize(dnz,onz);

1084:   MatSetLocalToGlobalMappingBlock(J,ltog);

1086:   /*
1087:     For each node in the grid: we get the neighbors in the local (on processor ordering
1088:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1089:     PETSc ordering.
1090:   */

1092:   for (i=xs; i<xs+nx; i++) {
1093:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1094:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1095:     for (j=ys; j<ys+ny; j++) {
1096:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1097:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1098:       for (k=zs; k<zs+nz; k++) {
1099:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1100:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1101: 
1102:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1103: 
1104:         cnt  = 0;
1105:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1106:           for (ii=istart; ii<iend+1; ii++) {
1107:             for (jj=jstart; jj<jend+1; jj++) {
1108:               for (kk=kstart; kk<kend+1; kk++) {
1109:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1110:               }
1111:             }
1112:           }
1113:         } else {  /* Star stencil */
1114:           cnt  = 0;
1115:           for (ii=istart; ii<iend+1; ii++) {
1116:             if (ii) {
1117:               /* jj and kk must be zero */
1118:               cols[cnt++]  = slot + ii;
1119:             } else {
1120:               for (jj=jstart; jj<jend+1; jj++) {
1121:                 if (jj) {
1122:                   /* ii and kk must be zero */
1123:                   cols[cnt++]  = slot + gnx*jj;
1124:                 } else {
1125:                   /* ii and jj must be zero */
1126:                   for (kk=kstart; kk<kend+1; kk++) {
1127:                     cols[cnt++]  = slot + gnx*gny*kk;
1128:                   }
1129:                 }
1130:               }
1131:             }
1132:           }
1133:         }
1134:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1135:       }
1136:     }
1137:   }
1138:   PetscFree(values);
1139:   PetscFree(cols);
1140:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1141:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1142:   return(0);
1143: }

1145: /* BAD! Almost identical to the BAIJ one */
1148: PetscErrorCode DAGetMatrix3d_MPISBAIJ(DA da,Mat J)
1149: {
1150:   PetscErrorCode         ierr;
1151:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1152:   PetscInt               m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1153:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1154:   MPI_Comm               comm;
1155:   PetscScalar            *values;
1156:   DAPeriodicType         wrap;
1157:   DAStencilType          st;
1158:   ISLocalToGlobalMapping ltog;

1161:   /*     
1162:          nc - number of components per grid point 
1163:          col - number of colors needed in one direction for single component problem
1164:   
1165:   */
1166:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1167:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1168:   col    = 2*s + 1;

1170:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1171:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1172:   PetscObjectGetComm((PetscObject)da,&comm);

1174:   /* create the matrix */
1175:   PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1176:   PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1177:   PetscMalloc(col*col*col*sizeof(PetscInt),&cols);

1179:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

1181:   /* determine the matrix preallocation information */
1182:   MatPreallocateSymmetricInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1183:   for (i=xs; i<xs+nx; i++) {
1184:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1185:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1186:     for (j=ys; j<ys+ny; j++) {
1187:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1188:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1189:       for (k=zs; k<zs+nz; k++) {
1190:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1191:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

1193:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1195:         /* Find block columns in block row */
1196:         cnt  = 0;
1197:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1198:           for (ii=istart; ii<iend+1; ii++) {
1199:             for (jj=jstart; jj<jend+1; jj++) {
1200:               for (kk=kstart; kk<kend+1; kk++) {
1201:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1202:               }
1203:             }
1204:           }
1205:         } else {  /* Star stencil */
1206:           cnt  = 0;
1207:           for (ii=istart; ii<iend+1; ii++) {
1208:             if (ii) {
1209:               /* jj and kk must be zero */
1210:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
1211:               cols[cnt++]  = slot + ii;
1212:             } else {
1213:               for (jj=jstart; jj<jend+1; jj++) {
1214:                 if (jj) {
1215:                   /* ii and kk must be zero */
1216:                   cols[cnt++]  = slot + gnx*jj;
1217:                 } else {
1218:                   /* ii and jj must be zero */
1219:                   for (kk=kstart; kk<kend+1; kk++) {
1220:                     cols[cnt++]  = slot + gnx*gny*kk;
1221:                   }
1222:                 }
1223:               }
1224:             }
1225:           }
1226:         }
1227:         MatPreallocateSymmetricSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1228:       }
1229:     }
1230:   }
1231:   MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1232:   MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1233:   MatPreallocateFinalize(dnz,onz);

1235:   MatSetLocalToGlobalMappingBlock(J,ltog);

1237:   /*
1238:     For each node in the grid: we get the neighbors in the local (on processor ordering
1239:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1240:     PETSc ordering.
1241:   */

1243:   for (i=xs; i<xs+nx; i++) {
1244:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1245:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1246:     for (j=ys; j<ys+ny; j++) {
1247:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1248:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1249:       for (k=zs; k<zs+nz; k++) {
1250:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1251:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1252: 
1253:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1254: 
1255:         cnt  = 0;
1256:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1257:           for (ii=istart; ii<iend+1; ii++) {
1258:             for (jj=jstart; jj<jend+1; jj++) {
1259:               for (kk=kstart; kk<kend+1; kk++) {
1260:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1261:               }
1262:             }
1263:           }
1264:         } else {  /* Star stencil */
1265:           cnt  = 0;
1266:           for (ii=istart; ii<iend+1; ii++) {
1267:             if (ii) {
1268:               /* jj and kk must be zero */
1269:               cols[cnt++]  = slot + ii;
1270:             } else {
1271:               for (jj=jstart; jj<jend+1; jj++) {
1272:                 if (jj) {
1273:                   /* ii and kk must be zero */
1274:                   cols[cnt++]  = slot + gnx*jj;
1275:                 } else {
1276:                   /* ii and jj must be zero */
1277:                   for (kk=kstart; kk<kend+1; kk++) {
1278:                     cols[cnt++]  = slot + gnx*gny*kk;
1279:                   }
1280:                 }
1281:               }
1282:             }
1283:           }
1284:         }
1285:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1286:       }
1287:     }
1288:   }
1289:   PetscFree(values);
1290:   PetscFree(cols);
1291:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1292:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1293:   return(0);
1294: }


1297: /* ---------------------------------------------------------------------------------*/

1301: PetscErrorCode DAGetMatrix3d_MPIAIJ_Fill(DA da,Mat J)
1302: {
1303:   PetscErrorCode         ierr;
1304:   PetscInt               xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1305:   PetscInt               m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
1306:   PetscInt               istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1307:   PetscInt               ifill_col,*dfill = da->dfill,*ofill = da->ofill;
1308:   MPI_Comm               comm;
1309:   PetscScalar            *values;
1310:   DAPeriodicType         wrap;
1311:   ISLocalToGlobalMapping ltog;
1312:   DAStencilType          st;

1315:   /*     
1316:          nc - number of components per grid point 
1317:          col - number of colors needed in one direction for single component problem
1318:   
1319:   */
1320:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1321:   col    = 2*s + 1;
1322:   if (DAXPeriodic(wrap) && (m % col)){
1323:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1324:                  by 2*stencil_width + 1\n");
1325:   }
1326:   if (DAYPeriodic(wrap) && (n % col)){
1327:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1328:                  by 2*stencil_width + 1\n");
1329:   }
1330:   if (DAZPeriodic(wrap) && (p % col)){
1331:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1332:                  by 2*stencil_width + 1\n");
1333:   }

1335:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1336:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1337:   PetscObjectGetComm((PetscObject)da,&comm);

1339:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1340:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1341:   PetscMalloc(nc*sizeof(PetscInt),&rows);
1342:   PetscMalloc(col*col*col*nc*sizeof(PetscInt),&cols);
1343:   DAGetISLocalToGlobalMapping(da,&ltog);

1345:   /* determine the matrix preallocation information */
1346:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);


1349:   for (i=xs; i<xs+nx; i++) {
1350:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1351:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1352:     for (j=ys; j<ys+ny; j++) {
1353:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1354:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1355:       for (k=zs; k<zs+nz; k++) {
1356:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1357:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1358: 
1359:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1360: 
1361:         for (l=0; l<nc; l++) {
1362:           cnt  = 0;
1363:           for (ii=istart; ii<iend+1; ii++) {
1364:             for (jj=jstart; jj<jend+1; jj++) {
1365:               for (kk=kstart; kk<kend+1; kk++) {
1366:                 if (ii || jj || kk) {
1367:                   if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1368:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1369:                       cols[cnt++]  = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1370:                   }
1371:                 } else {
1372:                   if (dfill) {
1373:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1374:                       cols[cnt++]  = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1375:                   } else {
1376:                     for (ifill_col=0; ifill_col<nc; ifill_col++)
1377:                       cols[cnt++]  = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1378:                   }
1379:                 }
1380:               }
1381:             }
1382:           }
1383:           rows[0] = l + nc*(slot);
1384:           MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
1385:         }
1386:       }
1387:     }
1388:   }
1389:   MatSeqAIJSetPreallocation(J,0,dnz);
1390:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1391:   MatPreallocateFinalize(dnz,onz);
1392:   MatSetBlockSize(J,nc);
1393:   MatSetLocalToGlobalMapping(J,ltog);

1395:   /*
1396:     For each node in the grid: we get the neighbors in the local (on processor ordering
1397:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1398:     PETSc ordering.
1399:   */
1400:   for (i=xs; i<xs+nx; i++) {
1401:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1402:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1403:     for (j=ys; j<ys+ny; j++) {
1404:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1405:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1406:       for (k=zs; k<zs+nz; k++) {
1407:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1408:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1409: 
1410:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1411: 
1412:         for (l=0; l<nc; l++) {
1413:           cnt  = 0;
1414:           for (ii=istart; ii<iend+1; ii++) {
1415:             for (jj=jstart; jj<jend+1; jj++) {
1416:               for (kk=kstart; kk<kend+1; kk++) {
1417:                 if (ii || jj || kk) {
1418:                   if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1419:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1420:                       cols[cnt++]  = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1421:                   }
1422:                 } else {
1423:                   if (dfill) {
1424:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1425:                       cols[cnt++]  = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1426:                   } else {
1427:                     for (ifill_col=0; ifill_col<nc; ifill_col++)
1428:                       cols[cnt++]  = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1429:                   }
1430:                 }
1431:               }
1432:             }
1433:           }
1434:           rows[0] = l + nc*(slot);
1435:           MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
1436:         }
1437:       }
1438:     }
1439:   }
1440:   PetscFree(values);
1441:   PetscFree(rows);
1442:   PetscFree(cols);
1443:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1444:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1445:   return(0);
1446: }