Actual source code: fdda.c

  1: /*$Id: fdda.c,v 1.75 2001/08/07 21:31:51 bsmith Exp $*/
  2: 
 3:  #include src/dm/da/daimpl.h
 4:  #include petscmat.h


  7: EXTERN int DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
  8: EXTERN int DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
  9: EXTERN int DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
 10: EXTERN int DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);

 12: /*
 13:    For ghost i that may be negative or greater than the upper bound this
 14:   maps it into the 0:m-1 range using periodicity
 15: */
 16: #define SetInRange(i,m) ((i < 0) ? m+i:((i >= m) ? i-m:i))

 20: static int DASetBlockFills_Private(int *dfill,int w,int **rfill)
 21: {
 22:   int ierr,i,j,nz,*fill;

 25:   /* count number nonzeros */
 26:   nz = 0;
 27:   for (i=0; i<w; i++) {
 28:     for (j=0; j<w; j++) {
 29:       if (dfill[w*i+j]) nz++;
 30:     }
 31:   }
 32:   PetscMalloc((nz + w + 1)*sizeof(int),&fill);
 33:   /* construct modified CSR storage of nonzero structure */
 34:   nz = w + 1;
 35:   for (i=0; i<w; i++) {
 36:     fill[i] = nz;
 37:     for (j=0; j<w; j++) {
 38:       if (dfill[w*i+j]) {
 39:         fill[nz] = j;
 40:         nz++;
 41:       }
 42:     }
 43:   }
 44:   fill[w] = nz;
 45: 
 46:   *rfill = fill;
 47:   return(0);
 48: }

 52: /*@C
 53:     DASetBlockFills - Sets the fill pattern in each block for a multi-component problem
 54:     of the matrix returned by DAGetMatrix().

 56:     Collective on DA

 58:     Input Parameter:
 59: +   da - the distributed array
 60: .   dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
 61: -   ofill - the fill pattern in the off-diagonal blocks


 64:     Level: developer

 66:     Notes: This only makes sense when you are doing multicomponent problems but using the
 67:        MPIAIJ matrix format

 69:            The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
 70:        representing coupling and 0 entries for missing coupling. For example 
 71: $             dfill[3][3] = {1, 0, 0,
 72: $                            1, 1, 0,
 73: $                            0, 1, 1} 
 74:        means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with 
 75:        itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the 
 76:        diagonal block.

 78:      DASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
 79:      can be represented in the dfill, ofill format

 81:    Contributed by Glenn Hammond

 83: .seealso DAGetMatrix(), DASetGetMatrix()

 85: @*/
 86: int DASetBlockFills(DA da,int *dfill,int *ofill)
 87: {

 91:   if (dfill) {
 92:     DASetBlockFills_Private(dfill,da->w,&da->dfill);
 93:   }
 94:   DASetBlockFills_Private(ofill,da->w,&da->ofill);
 95:   return(0);
 96: }


101: /*@C
102:     DAGetColoring - Gets the coloring required for computing the Jacobian via
103:     finite differences on a function defined using a stencil on the DA.

105:     Collective on DA

107:     Input Parameter:
108: +   da - the distributed array
109: -   ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED

111:     Output Parameters:
112: .   coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)

114:     Level: advanced

116:     Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used 
117:    for efficient (parallel or thread based) triangular solves etc is NOT yet 
118:    available. 


121: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring

123: @*/
124: int DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
125: {
126:   int        ierr,dim;

129:   /*
130:                                   m
131:           ------------------------------------------------------
132:          |                                                     |
133:          |                                                     |
134:          |               ----------------------                |
135:          |               |                    |                |
136:       n  |           yn  |                    |                |
137:          |               |                    |                |
138:          |               .---------------------                |
139:          |             (xs,ys)     xn                          |
140:          |            .                                        |
141:          |         (gxs,gys)                                   |
142:          |                                                     |
143:           -----------------------------------------------------
144:   */

146:   /*     
147:          nc - number of components per grid point 
148:          col - number of colors needed in one direction for single component problem
149:   
150:   */
151:   DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);

153:   /*
154:      We do not provide a getcoloring function in the DA operations because 
155:    the basic DA does not know about matrices. We think of DA as being more 
156:    more low-level then matrices.
157:   */
158:   if (dim == 1) {
159:     DAGetColoring1d_MPIAIJ(da,ctype,coloring);
160:   } else if (dim == 2) {
161:      DAGetColoring2d_MPIAIJ(da,ctype,coloring);
162:   } else if (dim == 3) {
163:      DAGetColoring3d_MPIAIJ(da,ctype,coloring);
164:   } else {
165:       SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
166:   }
167:   return(0);
168: }

170: /* ---------------------------------------------------------------------------------*/

174: int DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
175: {
176:   int                    ierr,xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny;
177:   int                    m,n,M,N,dim,s,k,nc,col,size;
178:   MPI_Comm               comm;
179:   DAPeriodicType         wrap;
180:   DAStencilType          st;
181:   ISColoringValue        *colors;

184:   /*     
185:          nc - number of components per grid point 
186:          col - number of colors needed in one direction for single component problem
187:   
188:   */
189:   DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&wrap,&st);
190:   col    = 2*s + 1;
191:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
192:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
193:   PetscObjectGetComm((PetscObject)da,&comm);
194:   MPI_Comm_size(comm,&size);

196:   /* special case as taught to us by Paul Hovland */
197:   if (st == DA_STENCIL_STAR && s == 1) {
198:     DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
199:   } else {

201:     if (DAXPeriodic(wrap) && (m % col)){
202:       SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
203:                  by 2*stencil_width + 1\n");
204:     }
205:     if (DAYPeriodic(wrap) && (n % col)){
206:       SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
207:                  by 2*stencil_width + 1\n");
208:     }
209:     if (ctype == IS_COLORING_LOCAL) {
210:       if (!da->localcoloring) {
211:         PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
212:         ii = 0;
213:         for (j=ys; j<ys+ny; j++) {
214:           for (i=xs; i<xs+nx; i++) {
215:             for (k=0; k<nc; k++) {
216:               colors[ii++] = k + nc*((i % col) + col*(j % col));
217:             }
218:           }
219:         }
220:         ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
221:       }
222:       *coloring = da->localcoloring;
223:     } else if (ctype == IS_COLORING_GHOSTED) {
224:       if (!da->ghostedcoloring) {
225:         PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
226:         ii = 0;
227:         for (j=gys; j<gys+gny; j++) {
228:           for (i=gxs; i<gxs+gnx; i++) {
229:             for (k=0; k<nc; k++) {
230:               /* the complicated stuff is to handle periodic boundaries */
231:               colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
232:             }
233:           }
234:         }
235:         ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
236:         ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
237:       }
238:       *coloring = da->ghostedcoloring;
239:     } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
240:   }
241:   ISColoringReference(*coloring);
242:   return(0);
243: }

245: /* ---------------------------------------------------------------------------------*/

249: int DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
250: {
251:   int                    ierr,xs,ys,nx,ny,i,j,gxs,gys,gnx,gny;
252:   int                    m,n,p,dim,s,k,nc,col,size,zs,gzs,ii,l,nz,gnz,M,N,P;
253:   MPI_Comm               comm;
254:   DAPeriodicType         wrap;
255:   DAStencilType          st;
256:   ISColoringValue        *colors;

259:   /*     
260:          nc - number of components per grid point 
261:          col - number of colors needed in one direction for single component problem
262:   
263:   */
264:   DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
265:   col    = 2*s + 1;
266:   if (DAXPeriodic(wrap) && (m % col)){
267:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
268:                  by 2*stencil_width + 1\n");
269:   }
270:   if (DAYPeriodic(wrap) && (n % col)){
271:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
272:                  by 2*stencil_width + 1\n");
273:   }
274:   if (DAZPeriodic(wrap) && (p % col)){
275:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
276:                  by 2*stencil_width + 1\n");
277:   }

279:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
280:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
281:   PetscObjectGetComm((PetscObject)da,&comm);
282:   MPI_Comm_size(comm,&size);

284:   /* create the coloring */
285:   if (ctype == IS_COLORING_LOCAL) {
286:     if (!da->localcoloring) {
287:       PetscMalloc(nc*nx*ny*nz*sizeof(ISColoringValue),&colors);
288:       ii = 0;
289:       for (k=zs; k<zs+nz; k++) {
290:         for (j=ys; j<ys+ny; j++) {
291:           for (i=xs; i<xs+nx; i++) {
292:             for (l=0; l<nc; l++) {
293:               colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
294:             }
295:           }
296:         }
297:       }
298:       ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
299:     }
300:     *coloring = da->localcoloring;
301:   } else if (ctype == IS_COLORING_GHOSTED) {
302:     if (!da->ghostedcoloring) {
303:       PetscMalloc(nc*gnx*gny*gnz*sizeof(ISColoringValue),&colors);
304:       ii = 0;
305:       for (k=gzs; k<gzs+gnz; k++) {
306:         for (j=gys; j<gys+gny; j++) {
307:           for (i=gxs; i<gxs+gnx; i++) {
308:             for (l=0; l<nc; l++) {
309:               /* the complicated stuff is to handle periodic boundaries */
310:               colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
311:             }
312:           }
313:         }
314:       }
315:       ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
316:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
317:     }
318:     *coloring = da->ghostedcoloring;
319:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
320:   ISColoringReference(*coloring);
321:   return(0);
322: }

324: /* ---------------------------------------------------------------------------------*/

328: int DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
329: {
330:   int                    ierr,xs,nx,i,i1,gxs,gnx,l;
331:   int                    m,M,dim,s,nc,col,size;
332:   MPI_Comm               comm;
333:   DAPeriodicType         wrap;
334:   ISColoringValue        *colors;

337:   /*     
338:          nc - number of components per grid point 
339:          col - number of colors needed in one direction for single component problem
340:   
341:   */
342:   DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
343:   col    = 2*s + 1;

345:   if (DAXPeriodic(wrap) && (m % col)) {
346:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisible\n\
347:                  by 2*stencil_width + 1\n");
348:   }

350:   DAGetCorners(da,&xs,0,0,&nx,0,0);
351:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
352:   PetscObjectGetComm((PetscObject)da,&comm);
353:   MPI_Comm_size(comm,&size);

355:   /* create the coloring */
356:   if (ctype == IS_COLORING_LOCAL) {
357:     if (!da->localcoloring) {
358:       PetscMalloc(nc*nx*sizeof(ISColoringValue),&colors);
359:       i1 = 0;
360:       for (i=xs; i<xs+nx; i++) {
361:         for (l=0; l<nc; l++) {
362:           colors[i1++] = l + nc*(i % col);
363:         }
364:       }
365:       ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
366:     }
367:     *coloring = da->localcoloring;
368:   } else if (ctype == IS_COLORING_GHOSTED) {
369:     if (!da->ghostedcoloring) {
370:       PetscMalloc(nc*gnx*sizeof(ISColoringValue),&colors);
371:       i1 = 0;
372:       for (i=gxs; i<gxs+gnx; i++) {
373:         for (l=0; l<nc; l++) {
374:           /* the complicated stuff is to handle periodic boundaries */
375:           colors[i1++] = l + nc*(SetInRange(i,m) % col);
376:         }
377:       }
378:       ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
379:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
380:     }
381:     *coloring = da->ghostedcoloring;
382:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
383:   ISColoringReference(*coloring);
384:   return(0);
385: }

389: int DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
390: {
391:   int            ierr,xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny;
392:   int            m,n,dim,s,k,nc;
393:   MPI_Comm       comm;
394:   DAPeriodicType wrap;
395:   ISColoringValue        *colors;

398:   /*     
399:          nc - number of components per grid point 
400:          col - number of colors needed in one direction for single component problem
401:   
402:   */
403:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,0);
404:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
405:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
406:   PetscObjectGetComm((PetscObject)da,&comm);

408:   if (DAXPeriodic(wrap) && (m % 5)){
409:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
410:                  by 5\n");
411:   }
412:   if (DAYPeriodic(wrap) && (n % 5)){
413:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
414:                  by 5\n");
415:   }

417:   /* create the coloring */
418:   if (ctype == IS_COLORING_LOCAL) {
419:     if (!da->localcoloring) {
420:       PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
421:       ii = 0;
422:       for (j=ys; j<ys+ny; j++) {
423:         for (i=xs; i<xs+nx; i++) {
424:           for (k=0; k<nc; k++) {
425:             colors[ii++] = k + nc*((3*j+i) % 5);
426:           }
427:         }
428:       }
429:       ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
430:     }
431:     *coloring = da->localcoloring;
432:   } else if (ctype == IS_COLORING_GHOSTED) {
433:     if (!da->ghostedcoloring) {
434:       PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
435:       ii = 0;
436:       for (j=gys; j<gys+gny; j++) {
437:         for (i=gxs; i<gxs+gnx; i++) {
438:           for (k=0; k<nc; k++) {
439:             colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
440:           }
441:         }
442:       }
443:       ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
444:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
445:     }
446:     *coloring = da->ghostedcoloring;
447:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
448:   return(0);
449: }

451: /* =========================================================================== */
452: EXTERN int DAGetMatrix1d_MPIAIJ(DA,Mat);
453: EXTERN int DAGetMatrix2d_MPIAIJ(DA,Mat);
454: EXTERN int DAGetMatrix2d_MPIAIJ_Fill(DA,Mat);
455: EXTERN int DAGetMatrix3d_MPIAIJ(DA,Mat);
456: EXTERN int DAGetMatrix3d_MPIAIJ_Fill(DA,Mat);
457: EXTERN int DAGetMatrix3d_MPIBAIJ(DA,Mat);
458: EXTERN int DAGetMatrix3d_MPISBAIJ(DA,Mat);

462: /*@C
463:     DAGetMatrix - Creates a matrix with the correct parallel layout and nonzero structure required for 
464:       computing the Jacobian on a function defined using the stencil set in the DA.

466:     Collective on DA

468:     Input Parameter:
469: +   da - the distributed array
470: -   mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, or
471:             any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).

473:     Output Parameters:
474: .   J  - matrix with the correct nonzero structure
475:         (obviously without the correct Jacobian values)

477:     Level: advanced

479:     Notes: This properly preallocates the number of nonzeros in the sparse matrix so you 
480:        do not need to do it yourself.

482: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()

484: @*/
485: int DAGetMatrix(DA da,MatType mtype,Mat *J)
486: {
487:   int      ierr,dim,dof,nx,ny,nz;
488:   Mat      A;
489:   MPI_Comm comm;
490:   MatType  Atype;
491:   void     (*aij)(void)=PETSC_NULL,(*baij)(void)=PETSC_NULL,(*sbaij)(void)=PETSC_NULL;

494:   /*
495:                                   m
496:           ------------------------------------------------------
497:          |                                                     |
498:          |                                                     |
499:          |               ----------------------                |
500:          |               |                    |                |
501:       n  |           yn  |                    |                |
502:          |               |                    |                |
503:          |               .---------------------                |
504:          |             (xs,ys)     xn                          |
505:          |            .                                        |
506:          |         (gxs,gys)                                   |
507:          |                                                     |
508:           -----------------------------------------------------
509:   */

511:   /*     
512:          nc - number of components per grid point 
513:          col - number of colors needed in one direction for single component problem
514:   
515:   */
516:   DAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0);
517:   DAGetCorners(da,0,0,0,&nx,&ny,&nz);
518:   PetscObjectGetComm((PetscObject)da,&comm);
519:   MatCreate(comm,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,&A);
520:   MatSetType(A,mtype);
521:   MatSetFromOptions(A);
522:   MatGetType(A,&Atype);
523:   /*
524:      We do not provide a getmatrix function in the DA operations because 
525:    the basic DA does not know about matrices. We think of DA as being more 
526:    more low-level then matrices. This is kind of cheating but, cause sometimes 
527:    we think of DA has higher level then matrices.

529:      We could switch based on Atype (or mtype), but we do not since the
530:    specialized setting routines depend only the particular preallocation
531:    details of the matrix, not the type itself.
532:   */
533:   PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
534:   if (!aij) {
535:     PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
536:   }
537:   if (aij) {
538:     if (dim == 1) {
539:       DAGetMatrix1d_MPIAIJ(da,A);
540:     } else if (dim == 2) {
541:       if (da->ofill) {
542:         DAGetMatrix2d_MPIAIJ_Fill(da,A);
543:       } else {
544:         DAGetMatrix2d_MPIAIJ(da,A);
545:       }
546:     } else if (dim == 3) {
547:       if (da->ofill) {
548:         DAGetMatrix3d_MPIAIJ_Fill(da,A);
549:       } else {
550:         DAGetMatrix3d_MPIAIJ(da,A);
551:       }
552:     }
553:   } else if (dim == 3) {
554:     PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
555:     if (!baij) {
556:       PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
557:     }
558:     if (baij) {
559:       DAGetMatrix3d_MPIBAIJ(da,A);
560:     } else {
561:       PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
562:       if (!sbaij) {
563:         PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
564:       }
565:       if (sbaij) {
566:         DAGetMatrix3d_MPISBAIJ(da,A);
567:       } else {
568:         SETERRQ2(PETSC_ERR_SUP,"Not implemented for %d dimension and Matrix Type: %s!\n" \
569:                                "Send mail to petsc-maint@mcs.anl.gov for code",
570:                                dim,Atype);
571:       }
572:     }
573:   } else {
574:     SETERRQ2(PETSC_ERR_SUP,"Not implemented for %d dimension and matrix type: %s!\n" \
575:                            "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype);
576:   }
577:   *J = A;
578:   return(0);
579: }

581: /* ---------------------------------------------------------------------------------*/
584: int DAGetMatrix2d_MPIAIJ(DA da,Mat J)
585: {
586:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
587:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
588:   int                    lstart,lend,pstart,pend,*dnz,*onz;
589:   int                    dims[2],starts[2];
590:   MPI_Comm               comm;
591:   PetscScalar            *values;
592:   DAPeriodicType         wrap;
593:   ISLocalToGlobalMapping ltog;
594:   DAStencilType          st;

597:   /*     
598:          nc - number of components per grid point 
599:          col - number of colors needed in one direction for single component problem
600:   
601:   */
602:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
603:   col = 2*s + 1;
604:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
605:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
606:   PetscObjectGetComm((PetscObject)da,&comm);

608:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
609:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
610:   PetscMalloc(nc*sizeof(int),&rows);
611:   PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
612:   DAGetISLocalToGlobalMapping(da,&ltog);
613: 
614:   /* determine the matrix preallocation information */
615:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
616:   for (i=xs; i<xs+nx; i++) {

618:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
619:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

621:     for (j=ys; j<ys+ny; j++) {
622:       slot = i - gxs + gnx*(j - gys);

624:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
625:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

627:       cnt  = 0;
628:       for (k=0; k<nc; k++) {
629:         for (l=lstart; l<lend+1; l++) {
630:           for (p=pstart; p<pend+1; p++) {
631:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
632:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
633:             }
634:           }
635:         }
636:         rows[k] = k + nc*(slot);
637:       }
638:       MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
639:     }
640:   }
641:   MatSeqAIJSetPreallocation(J,0,dnz);
642:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
643:   MatPreallocateFinalize(dnz,onz);

645:   MatSetLocalToGlobalMapping(J,ltog);
646:   DAGetGhostCorners(da,&starts[0],&starts[1],PETSC_IGNORE,&dims[0],&dims[1],PETSC_IGNORE);
647:   MatSetStencil(J,2,dims,starts,nc);

649:   /*
650:     For each node in the grid: we get the neighbors in the local (on processor ordering
651:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
652:     PETSc ordering.
653:   */
654:   for (i=xs; i<xs+nx; i++) {
655: 
656:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
657:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
658: 
659:     for (j=ys; j<ys+ny; j++) {
660:       slot = i - gxs + gnx*(j - gys);
661: 
662:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
663:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

665:       cnt  = 0;
666:       for (k=0; k<nc; k++) {
667:         for (l=lstart; l<lend+1; l++) {
668:           for (p=pstart; p<pend+1; p++) {
669:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
670:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
671:             }
672:           }
673:         }
674:         rows[k]      = k + nc*(slot);
675:       }
676:       MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
677:     }
678:   }
679:   PetscFree(values);
680:   PetscFree(rows);
681:   PetscFree(cols);
682:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
683:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
684:   return(0);
685: }

689: int DAGetMatrix2d_MPIAIJ_Fill(DA da,Mat J)
690: {
691:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
692:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
693:   int                    lstart,lend,pstart,pend,*dnz,*onz;
694:   int                    dims[2],starts[2],ifill_col,*ofill = da->ofill, *dfill = da->dfill;
695:   MPI_Comm               comm;
696:   PetscScalar            *values;
697:   DAPeriodicType         wrap;
698:   ISLocalToGlobalMapping ltog;
699:   DAStencilType          st;

702:   /*     
703:          nc - number of components per grid point 
704:          col - number of colors needed in one direction for single component problem
705:   
706:   */
707:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
708:   col = 2*s + 1;
709:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
710:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
711:   PetscObjectGetComm((PetscObject)da,&comm);

713:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
714:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
715:   PetscMalloc(nc*sizeof(int),&rows);
716:   PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
717:   DAGetISLocalToGlobalMapping(da,&ltog);
718: 
719:   /* determine the matrix preallocation information */
720:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
721:   for (i=xs; i<xs+nx; i++) {

723:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
724:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

726:     for (j=ys; j<ys+ny; j++) {
727:       slot = i - gxs + gnx*(j - gys);

729:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
730:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

732:       for (k=0; k<nc; k++) {
733:         cnt  = 0;
734:         for (l=lstart; l<lend+1; l++) {
735:           for (p=pstart; p<pend+1; p++) {
736:             if (l || p) {
737:               if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
738:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
739:                   cols[cnt++]  = ofill[ifill_col] + nc*(slot + gnx*l + p);
740:               }
741:             } else {
742:               if (dfill) {
743:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
744:                   cols[cnt++]  = dfill[ifill_col] + nc*(slot + gnx*l + p);
745:               } else {
746:                 for (ifill_col=0; ifill_col<nc; ifill_col++)
747:                   cols[cnt++]  = ifill_col + nc*(slot + gnx*l + p);
748:               }
749:             }
750:           }
751:         }
752:         rows[0] = k + nc*(slot);
753:         MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
754:       }
755:     }
756:   }
757:   MatSeqAIJSetPreallocation(J,0,dnz);
758:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
759:   MatPreallocateFinalize(dnz,onz);

761:   MatSetLocalToGlobalMapping(J,ltog);
762:   DAGetGhostCorners(da,&starts[0],&starts[1],PETSC_IGNORE,&dims[0],&dims[1],PETSC_IGNORE);
763:   MatSetStencil(J,2,dims,starts,nc);

765:   /*
766:     For each node in the grid: we get the neighbors in the local (on processor ordering
767:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
768:     PETSc ordering.
769:   */
770:   for (i=xs; i<xs+nx; i++) {
771: 
772:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
773:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
774: 
775:     for (j=ys; j<ys+ny; j++) {
776:       slot = i - gxs + gnx*(j - gys);
777: 
778:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
779:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

781:       for (k=0; k<nc; k++) {
782:         cnt  = 0;
783:         for (l=lstart; l<lend+1; l++) {
784:           for (p=pstart; p<pend+1; p++) {
785:             if (l || p) {
786:               if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
787:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
788:                   cols[cnt++]  = ofill[ifill_col] + nc*(slot + gnx*l + p);
789:               }
790:             } else {
791:               if (dfill) {
792:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
793:                   cols[cnt++]  = dfill[ifill_col] + nc*(slot + gnx*l + p);
794:               } else {
795:                 for (ifill_col=0; ifill_col<nc; ifill_col++)
796:                   cols[cnt++]  = ifill_col + nc*(slot + gnx*l + p);
797:               }
798:             }
799:           }
800:         }
801:         rows[0]      = k + nc*(slot);
802:         MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
803:       }
804:     }
805:   }
806:   PetscFree(values);
807:   PetscFree(rows);
808:   PetscFree(cols);
809:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
810:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
811:   return(0);
812: }

814: /* ---------------------------------------------------------------------------------*/

818: int DAGetMatrix3d_MPIAIJ(DA da,Mat J)
819: {
820:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
821:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
822:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
823:   int                    dims[3],starts[3];
824:   MPI_Comm               comm;
825:   PetscScalar            *values;
826:   DAPeriodicType         wrap;
827:   ISLocalToGlobalMapping ltog;
828:   DAStencilType          st;

831:   /*     
832:          nc - number of components per grid point 
833:          col - number of colors needed in one direction for single component problem
834:   
835:   */
836:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
837:   col    = 2*s + 1;

839:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
840:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
841:   PetscObjectGetComm((PetscObject)da,&comm);

843:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
844:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
845:   PetscMalloc(nc*sizeof(int),&rows);
846:   PetscMalloc(col*col*col*nc*sizeof(int),&cols);
847:   DAGetISLocalToGlobalMapping(da,&ltog);

849:   /* determine the matrix preallocation information */
850:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
851:   for (i=xs; i<xs+nx; i++) {
852:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
853:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
854:     for (j=ys; j<ys+ny; j++) {
855:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
856:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
857:       for (k=zs; k<zs+nz; k++) {
858:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
859:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
860: 
861:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
862: 
863:         cnt  = 0;
864:         for (l=0; l<nc; l++) {
865:           for (ii=istart; ii<iend+1; ii++) {
866:             for (jj=jstart; jj<jend+1; jj++) {
867:               for (kk=kstart; kk<kend+1; kk++) {
868:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
869:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
870:                 }
871:               }
872:             }
873:           }
874:           rows[l] = l + nc*(slot);
875:         }
876:         MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
877:       }
878:     }
879:   }
880:   MatSeqAIJSetPreallocation(J,0,dnz);
881:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
882:   MatPreallocateFinalize(dnz,onz);

884:   MatSetLocalToGlobalMapping(J,ltog);
885:   DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
886:   MatSetStencil(J,3,dims,starts,nc);

888:   /*
889:     For each node in the grid: we get the neighbors in the local (on processor ordering
890:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
891:     PETSc ordering.
892:   */
893:   for (i=xs; i<xs+nx; i++) {
894:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
895:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
896:     for (j=ys; j<ys+ny; j++) {
897:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
898:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
899:       for (k=zs; k<zs+nz; k++) {
900:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
901:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
902: 
903:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
904: 
905:         cnt  = 0;
906:         for (l=0; l<nc; l++) {
907:           for (ii=istart; ii<iend+1; ii++) {
908:             for (jj=jstart; jj<jend+1; jj++) {
909:               for (kk=kstart; kk<kend+1; kk++) {
910:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
911:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
912:                 }
913:               }
914:             }
915:           }
916:           rows[l]      = l + nc*(slot);
917:         }
918:         MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
919:       }
920:     }
921:   }
922:   PetscFree(values);
923:   PetscFree(rows);
924:   PetscFree(cols);
925:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
926:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
927:   return(0);
928: }

930: /* ---------------------------------------------------------------------------------*/

934: int DAGetMatrix1d_MPIAIJ(DA da,Mat J)
935: {
936:   int                    ierr,xs,nx,i,i1,slot,gxs,gnx;
937:   int                    m,dim,s,*cols,nc,*rows,col,cnt,l;
938:   int                    istart,iend;
939:   int                    dims[1],starts[1];
940:   PetscScalar            *values;
941:   DAPeriodicType         wrap;
942:   ISLocalToGlobalMapping ltog;

945:   /*     
946:          nc - number of components per grid point 
947:          col - number of colors needed in one direction for single component problem
948:   
949:   */
950:   DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
951:   col    = 2*s + 1;

953:   DAGetCorners(da,&xs,0,0,&nx,0,0);
954:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);

956:   MatSeqAIJSetPreallocation(J,col*nc,0);
957:   MatMPIAIJSetPreallocation(J,col*nc,0,0,0);

959:   DAGetGhostCorners(da,&starts[0],PETSC_IGNORE,PETSC_IGNORE,&dims[0],PETSC_IGNORE,PETSC_IGNORE);
960:   MatSetStencil(J,1,dims,starts,nc);
961: 
962:   PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
963:   PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
964:   PetscMalloc(nc*sizeof(int),&rows);
965:   PetscMalloc(col*nc*sizeof(int),&cols);
966: 
967:   DAGetISLocalToGlobalMapping(da,&ltog);
968:   MatSetLocalToGlobalMapping(J,ltog);
969: 
970:   /*
971:     For each node in the grid: we get the neighbors in the local (on processor ordering
972:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
973:     PETSc ordering.
974:   */
975:   for (i=xs; i<xs+nx; i++) {
976:     istart = PetscMax(-s,gxs - i);
977:     iend   = PetscMin(s,gxs + gnx - i - 1);
978:     slot   = i - gxs;
979: 
980:     cnt  = 0;
981:     for (l=0; l<nc; l++) {
982:       for (i1=istart; i1<iend+1; i1++) {
983:         cols[cnt++] = l + nc*(slot + i1);
984:       }
985:       rows[l]      = l + nc*(slot);
986:     }
987:     MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
988:   }
989:   PetscFree(values);
990:   PetscFree(rows);
991:   PetscFree(cols);
992:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
993:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
994:   return(0);
995: }

999: int DAGetMatrix3d_MPIBAIJ(DA da,Mat J)
1000: {
1001:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1002:   int                    m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1003:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1004:   MPI_Comm               comm;
1005:   PetscScalar            *values;
1006:   DAPeriodicType         wrap;
1007:   DAStencilType          st;
1008:   ISLocalToGlobalMapping ltog;

1011:   /*     
1012:          nc - number of components per grid point 
1013:          col - number of colors needed in one direction for single component problem
1014:   
1015:   */
1016:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1017:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1018:   col    = 2*s + 1;

1020:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1021:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1022:   PetscObjectGetComm((PetscObject)da,&comm);

1024:   PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1025:   PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1026:   PetscMalloc(col*col*col*sizeof(int),&cols);

1028:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

1030:   /* determine the matrix preallocation information */
1031:   MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1032:   for (i=xs; i<xs+nx; i++) {
1033:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1034:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1035:     for (j=ys; j<ys+ny; j++) {
1036:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1037:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1038:       for (k=zs; k<zs+nz; k++) {
1039:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1040:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

1042:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1044:         /* Find block columns in block row */
1045:         cnt  = 0;
1046:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1047:           for (ii=istart; ii<iend+1; ii++) {
1048:             for (jj=jstart; jj<jend+1; jj++) {
1049:               for (kk=kstart; kk<kend+1; kk++) {
1050:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1051:               }
1052:             }
1053:           }
1054:         } else {  /* Star stencil */
1055:           cnt  = 0;
1056:           for (ii=istart; ii<iend+1; ii++) {
1057:             if (ii) {
1058:               /* jj and kk must be zero */
1059:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
1060:               cols[cnt++]  = slot + ii;
1061:             } else {
1062:               for (jj=jstart; jj<jend+1; jj++) {
1063:                 if (jj) {
1064:                   /* ii and kk must be zero */
1065:                   cols[cnt++]  = slot + gnx*jj;
1066:                 } else {
1067:                   /* ii and jj must be zero */
1068:                   for (kk=kstart; kk<kend+1; kk++) {
1069:                     cols[cnt++]  = slot + gnx*gny*kk;
1070:                   }
1071:                 }
1072:               }
1073:             }
1074:           }
1075:         }
1076:         MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1077:       }
1078:     }
1079:   }
1080:   MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1081:   MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1082:   MatPreallocateFinalize(dnz,onz);

1084:   MatSetLocalToGlobalMappingBlock(J,ltog);

1086:   /*
1087:     For each node in the grid: we get the neighbors in the local (on processor ordering
1088:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1089:     PETSc ordering.
1090:   */

1092:   for (i=xs; i<xs+nx; i++) {
1093:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1094:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1095:     for (j=ys; j<ys+ny; j++) {
1096:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1097:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1098:       for (k=zs; k<zs+nz; k++) {
1099:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1100:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1101: 
1102:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1103: 
1104:         cnt  = 0;
1105:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1106:           for (ii=istart; ii<iend+1; ii++) {
1107:             for (jj=jstart; jj<jend+1; jj++) {
1108:               for (kk=kstart; kk<kend+1; kk++) {
1109:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1110:               }
1111:             }
1112:           }
1113:         } else {  /* Star stencil */
1114:           cnt  = 0;
1115:           for (ii=istart; ii<iend+1; ii++) {
1116:             if (ii) {
1117:               /* jj and kk must be zero */
1118:               cols[cnt++]  = slot + ii;
1119:             } else {
1120:               for (jj=jstart; jj<jend+1; jj++) {
1121:                 if (jj) {
1122:                   /* ii and kk must be zero */
1123:                   cols[cnt++]  = slot + gnx*jj;
1124:                 } else {
1125:                   /* ii and jj must be zero */
1126:                   for (kk=kstart; kk<kend+1; kk++) {
1127:                     cols[cnt++]  = slot + gnx*gny*kk;
1128:                   }
1129:                 }
1130:               }
1131:             }
1132:           }
1133:         }
1134:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1135:       }
1136:     }
1137:   }
1138:   PetscFree(values);
1139:   PetscFree(cols);
1140:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1141:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1142:   return(0);
1143: }

1145: /* BAD! Almost identical to the BAIJ one */
1148: int DAGetMatrix3d_MPISBAIJ(DA da,Mat J)
1149: {
1150:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1151:   int                    m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1152:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1153:   MPI_Comm               comm;
1154:   PetscScalar            *values;
1155:   DAPeriodicType         wrap;
1156:   DAStencilType          st;
1157:   ISLocalToGlobalMapping ltog;

1160:   /*     
1161:          nc - number of components per grid point 
1162:          col - number of colors needed in one direction for single component problem
1163:   
1164:   */
1165:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1166:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1167:   col    = 2*s + 1;

1169:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1170:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1171:   PetscObjectGetComm((PetscObject)da,&comm);

1173:   /* create the matrix */
1174:   PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1175:   PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1176:   PetscMalloc(col*col*col*sizeof(int),&cols);

1178:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

1180:   /* determine the matrix preallocation information */
1181:   MatPreallocateSymmetricInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1182:   for (i=xs; i<xs+nx; i++) {
1183:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1184:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1185:     for (j=ys; j<ys+ny; j++) {
1186:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1187:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1188:       for (k=zs; k<zs+nz; k++) {
1189:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1190:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

1192:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1194:         /* Find block columns in block row */
1195:         cnt  = 0;
1196:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1197:           for (ii=istart; ii<iend+1; ii++) {
1198:             for (jj=jstart; jj<jend+1; jj++) {
1199:               for (kk=kstart; kk<kend+1; kk++) {
1200:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1201:               }
1202:             }
1203:           }
1204:         } else {  /* Star stencil */
1205:           cnt  = 0;
1206:           for (ii=istart; ii<iend+1; ii++) {
1207:             if (ii) {
1208:               /* jj and kk must be zero */
1209:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
1210:               cols[cnt++]  = slot + ii;
1211:             } else {
1212:               for (jj=jstart; jj<jend+1; jj++) {
1213:                 if (jj) {
1214:                   /* ii and kk must be zero */
1215:                   cols[cnt++]  = slot + gnx*jj;
1216:                 } else {
1217:                   /* ii and jj must be zero */
1218:                   for (kk=kstart; kk<kend+1; kk++) {
1219:                     cols[cnt++]  = slot + gnx*gny*kk;
1220:                   }
1221:                 }
1222:               }
1223:             }
1224:           }
1225:         }
1226:         MatPreallocateSymmetricSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1227:       }
1228:     }
1229:   }
1230:   MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1231:   MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1232:   MatPreallocateFinalize(dnz,onz);

1234:   MatSetLocalToGlobalMappingBlock(J,ltog);

1236:   /*
1237:     For each node in the grid: we get the neighbors in the local (on processor ordering
1238:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1239:     PETSc ordering.
1240:   */

1242:   for (i=xs; i<xs+nx; i++) {
1243:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1244:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1245:     for (j=ys; j<ys+ny; j++) {
1246:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1247:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1248:       for (k=zs; k<zs+nz; k++) {
1249:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1250:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1251: 
1252:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1253: 
1254:         cnt  = 0;
1255:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1256:           for (ii=istart; ii<iend+1; ii++) {
1257:             for (jj=jstart; jj<jend+1; jj++) {
1258:               for (kk=kstart; kk<kend+1; kk++) {
1259:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1260:               }
1261:             }
1262:           }
1263:         } else {  /* Star stencil */
1264:           cnt  = 0;
1265:           for (ii=istart; ii<iend+1; ii++) {
1266:             if (ii) {
1267:               /* jj and kk must be zero */
1268:               cols[cnt++]  = slot + ii;
1269:             } else {
1270:               for (jj=jstart; jj<jend+1; jj++) {
1271:                 if (jj) {
1272:                   /* ii and kk must be zero */
1273:                   cols[cnt++]  = slot + gnx*jj;
1274:                 } else {
1275:                   /* ii and jj must be zero */
1276:                   for (kk=kstart; kk<kend+1; kk++) {
1277:                     cols[cnt++]  = slot + gnx*gny*kk;
1278:                   }
1279:                 }
1280:               }
1281:             }
1282:           }
1283:         }
1284:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1285:       }
1286:     }
1287:   }
1288:   PetscFree(values);
1289:   PetscFree(cols);
1290:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1291:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1292:   return(0);
1293: }


1296: /* ---------------------------------------------------------------------------------*/

1300: int DAGetMatrix3d_MPIAIJ_Fill(DA da,Mat J)
1301: {
1302:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1303:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
1304:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1305:   int                    ifill_col,*dfill = da->dfill,*ofill = da->ofill;
1306:   int                    dims[3],starts[3];
1307:   MPI_Comm               comm;
1308:   PetscScalar            *values;
1309:   DAPeriodicType         wrap;
1310:   ISLocalToGlobalMapping ltog;
1311:   DAStencilType          st;

1314:   /*     
1315:          nc - number of components per grid point 
1316:          col - number of colors needed in one direction for single component problem
1317:   
1318:   */
1319:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1320:   col    = 2*s + 1;
1321:   if (DAXPeriodic(wrap) && (m % col)){
1322:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1323:                  by 2*stencil_width + 1\n");
1324:   }
1325:   if (DAYPeriodic(wrap) && (n % col)){
1326:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1327:                  by 2*stencil_width + 1\n");
1328:   }
1329:   if (DAZPeriodic(wrap) && (p % col)){
1330:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1331:                  by 2*stencil_width + 1\n");
1332:   }

1334:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1335:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1336:   PetscObjectGetComm((PetscObject)da,&comm);

1338:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1339:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1340:   PetscMalloc(nc*sizeof(int),&rows);
1341:   PetscMalloc(col*col*col*nc*sizeof(int),&cols);
1342:   DAGetISLocalToGlobalMapping(da,&ltog);

1344:   /* determine the matrix preallocation information */
1345:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);


1348:   for (i=xs; i<xs+nx; i++) {
1349:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1350:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1351:     for (j=ys; j<ys+ny; j++) {
1352:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1353:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1354:       for (k=zs; k<zs+nz; k++) {
1355:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1356:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1357: 
1358:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1359: 
1360:         for (l=0; l<nc; l++) {
1361:           cnt  = 0;
1362:           for (ii=istart; ii<iend+1; ii++) {
1363:             for (jj=jstart; jj<jend+1; jj++) {
1364:               for (kk=kstart; kk<kend+1; kk++) {
1365:                 if (ii || jj || kk) {
1366:                   if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1367:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1368:                       cols[cnt++]  = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1369:                   }
1370:                 } else {
1371:                   if (dfill) {
1372:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1373:                       cols[cnt++]  = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1374:                   } else {
1375:                     for (ifill_col=0; ifill_col<nc; ifill_col++)
1376:                       cols[cnt++]  = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1377:                   }
1378:                 }
1379:               }
1380:             }
1381:           }
1382:           rows[0] = l + nc*(slot);
1383:           MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
1384:         }
1385:       }
1386:     }
1387:   }
1388:   MatSeqAIJSetPreallocation(J,0,dnz);
1389:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1390:   MatPreallocateFinalize(dnz,onz);

1392:   MatSetLocalToGlobalMapping(J,ltog);
1393:   DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
1394:   MatSetStencil(J,3,dims,starts,nc);

1396:   /*
1397:     For each node in the grid: we get the neighbors in the local (on processor ordering
1398:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1399:     PETSc ordering.
1400:   */
1401:   for (i=xs; i<xs+nx; i++) {
1402:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1403:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1404:     for (j=ys; j<ys+ny; j++) {
1405:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1406:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1407:       for (k=zs; k<zs+nz; k++) {
1408:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1409:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1410: 
1411:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1412: 
1413:         for (l=0; l<nc; l++) {
1414:           cnt  = 0;
1415:           for (ii=istart; ii<iend+1; ii++) {
1416:             for (jj=jstart; jj<jend+1; jj++) {
1417:               for (kk=kstart; kk<kend+1; kk++) {
1418:                 if (ii || jj || kk) {
1419:                   if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1420:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1421:                       cols[cnt++]  = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1422:                   }
1423:                 } else {
1424:                   if (dfill) {
1425:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1426:                       cols[cnt++]  = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1427:                   } else {
1428:                     for (ifill_col=0; ifill_col<nc; ifill_col++)
1429:                       cols[cnt++]  = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1430:                   }
1431:                 }
1432:               }
1433:             }
1434:           }
1435:           rows[0] = l + nc*(slot);
1436:           MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
1437:         }
1438:       }
1439:     }
1440:   }
1441:   PetscFree(values);
1442:   PetscFree(rows);
1443:   PetscFree(cols);
1444:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1445:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1446:   return(0);
1447: }