Actual source code: bjacobi.c

  1: /*
  2:    Defines a block Jacobi preconditioner.
  3: */
 4:  #include src/mat/matimpl.h
 5:  #include src/ksp/pc/pcimpl.h
 6:  #include src/ksp/pc/impls/bjacobi/bjacobi.h

  8: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
  9: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);

 13: static PetscErrorCode PCSetUp_BJacobi(PC pc)
 14: {
 15:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
 16:   Mat            mat = pc->mat,pmat = pc->pmat;
 17:   PetscErrorCode ierr,(*f)(Mat,PetscTruth*,MatReuse,Mat*);
 18:   PetscInt       N,M,start,i,sum,end;
 19:   PetscInt       bs,i_start=-1,i_end=-1;
 20:   PetscMPIInt    rank,size;
 21:   char           *pprefix,*mprefix;

 24:   MPI_Comm_rank(pc->comm,&rank);
 25:   MPI_Comm_size(pc->comm,&size);
 26:   MatGetLocalSize(pc->pmat,&M,&N);
 27:   MatGetBlockSize(pc->pmat,&bs);

 29:   /* ----------
 30:       Determines the number of blocks assigned to each processor 
 31:   */

 33:   /*   local block count  given */
 34:   if (jac->n_local > 0 && jac->n < 0) {
 35:     MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,pc->comm);
 36:     if (jac->l_lens) { /* check that user set these correctly */
 37:       sum = 0;
 38:       for (i=0; i<jac->n_local; i++) {
 39:         if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
 40:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 41:         }
 42:         sum += jac->l_lens[i];
 43:       }
 44:       if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
 45:     } else {
 46:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 47:       for (i=0; i<jac->n_local; i++) {
 48:         jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
 49:       }
 50:     }
 51:   } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
 52:     /* global blocks given: determine which ones are local */
 53:     if (jac->g_lens) {
 54:       /* check if the g_lens is has valid entries */
 55:       for (i=0; i<jac->n; i++) {
 56:         if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
 57:         if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
 58:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 59:         }
 60:       }
 61:       if (size == 1) {
 62:         jac->n_local = jac->n;
 63:         PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 64:         PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
 65:         /* check that user set these correctly */
 66:         sum = 0;
 67:         for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
 68:         if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
 69:       } else {
 70:         MatGetOwnershipRange(pc->pmat,&start,&end);
 71:         /* loop over blocks determing first one owned by me */
 72:         sum = 0;
 73:         for (i=0; i<jac->n+1; i++) {
 74:           if (sum == start) { i_start = i; goto start_1;}
 75:           if (i < jac->n) sum += jac->g_lens[i];
 76:         }
 77:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
 78:                    used in PCBJacobiSetTotalBlocks()\n\
 79:                    are not compatible with parallel matrix layout");
 80:  start_1:
 81:         for (i=i_start; i<jac->n+1; i++) {
 82:           if (sum == end) { i_end = i; goto end_1; }
 83:           if (i < jac->n) sum += jac->g_lens[i];
 84:         }
 85:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
 86:                       used in PCBJacobiSetTotalBlocks()\n\
 87:                       are not compatible with parallel matrix layout");
 88:  end_1:
 89:         jac->n_local = i_end - i_start;
 90:         PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 91:         PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
 92:       }
 93:     } else { /* no global blocks given, determine then using default layout */
 94:       jac->n_local = jac->n/size + ((jac->n % size) > rank);
 95:       PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
 96:       for (i=0; i<jac->n_local; i++) {
 97:         jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
 98:         if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
 99:       }
100:     }
101:   } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
102:     jac->n         = size;
103:     jac->n_local   = 1;
104:     PetscMalloc(sizeof(PetscInt),&jac->l_lens);
105:     jac->l_lens[0] = M;
106:   }

108:   MPI_Comm_size(pc->comm,&size);
109:   PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
110:   if (size == 1 && !f) {
111:     mat  = pc->mat;
112:     pmat = pc->pmat;
113:   } else {
114:     PetscTruth iscopy;
115:     MatReuse   scall;

117:     if (jac->use_true_local) {
118:       scall = MAT_INITIAL_MATRIX;
119:       if (pc->setupcalled) {
120:         if (pc->flag == SAME_NONZERO_PATTERN) {
121:           if (jac->tp_mat) {
122:             scall = MAT_REUSE_MATRIX;
123:             mat   = jac->tp_mat;
124:           }
125:         } else {
126:           if (jac->tp_mat)  {
127:             MatDestroy(jac->tp_mat);
128:           }
129:         }
130:       }
131:       if (!f) {
132:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
133:       }
134:       (*f)(pc->mat,&iscopy,scall,&mat);
135:       /* make submatrix have same prefix as entire matrix */
136:       PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
137:       PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
138:       if (iscopy) {
139:         jac->tp_mat = mat;
140:       }
141:     }
142:     if (pc->pmat != pc->mat || !jac->use_true_local) {
143:       scall = MAT_INITIAL_MATRIX;
144:       if (pc->setupcalled) {
145:         if (pc->flag == SAME_NONZERO_PATTERN) {
146:           if (jac->tp_pmat) {
147:             scall = MAT_REUSE_MATRIX;
148:             pmat   = jac->tp_pmat;
149:           }
150:         } else {
151:           if (jac->tp_pmat)  {
152:             MatDestroy(jac->tp_pmat);
153:           }
154:         }
155:       }
156:       PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
157:       if (!f) {
158:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
159:       }
160:       (*f)(pc->pmat,&iscopy,scall,&pmat);
161:       /* make submatrix have same prefix as entire matrix */
162:       PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
163:       PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
164:       if (iscopy) {
165:         jac->tp_pmat = pmat;
166:       }
167:     } else {
168:       pmat = mat;
169:     }
170:   }

172:   /* ------
173:      Setup code depends on the number of blocks 
174:   */
175:   if (jac->n_local == 1) {
176:     PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
177:   } else {
178:     PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
179:   }
180:   return(0);
181: }

183: /* Default destroy, if it has never been setup */
186: static PetscErrorCode PCDestroy_BJacobi(PC pc)
187: {
188:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

192:   if (jac->g_lens) {PetscFree(jac->g_lens);}
193:   if (jac->l_lens) {PetscFree(jac->l_lens);}
194:   PetscFree(jac);
195:   return(0);
196: }

200: static PetscErrorCode PCSetFromOptions_BJacobi(PC pc)
201: {
202:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
204:   PetscInt       blocks;
205:   PetscTruth     flg;

208:   PetscOptionsHead("Block Jacobi options");
209:     PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
210:     if (flg) {
211:       PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
212:     }
213:     PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
214:     if (flg) {
215:       PCBJacobiSetUseTrueLocal(pc);
216:     }
217:   PetscOptionsTail();
218:   return(0);
219: }

223: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
224: {
225:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
227:   PetscMPIInt    rank;
228:   PetscInt       i;
229:   PetscTruth     iascii,isstring;
230:   PetscViewer    sviewer;

233:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
234:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
235:   if (iascii) {
236:     if (jac->use_true_local) {
237:       PetscViewerASCIIPrintf(viewer,"  block Jacobi: using true local matrix, number of blocks = %D\n",jac->n);
238:     }
239:     PetscViewerASCIIPrintf(viewer,"  block Jacobi: number of blocks = %D\n",jac->n);
240:     MPI_Comm_rank(pc->comm,&rank);
241:     if (jac->same_local_solves) {
242:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
243:       PetscViewerGetSingleton(viewer,&sviewer);
244:       if (!rank && jac->ksp) {
245:         PetscViewerASCIIPushTab(viewer);
246:         KSPView(jac->ksp[0],sviewer);
247:         PetscViewerASCIIPopTab(viewer);
248:       }
249:       PetscViewerRestoreSingleton(viewer,&sviewer);
250:     } else {

252:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
253:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
254:                    rank,jac->n_local,jac->first_local);
255:       PetscViewerASCIIPushTab(viewer);
256:       for (i=0; i<jac->n_local; i++) {
257:         PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
258:         PetscViewerGetSingleton(viewer,&sviewer);
259:         KSPView(jac->ksp[i],sviewer);
260:         if (i != jac->n_local-1) {
261:           PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
262:         }
263:         PetscViewerRestoreSingleton(viewer,&sviewer);
264:       }
265:       PetscViewerASCIIPopTab(viewer);
266:       PetscViewerFlush(viewer);
267:     }
268:   } else if (isstring) {
269:     PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
270:     PetscViewerGetSingleton(viewer,&sviewer);
271:     if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
272:     PetscViewerRestoreSingleton(viewer,&sviewer);
273:   } else {
274:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
275:   }
276:   return(0);
277: }

279: /* -------------------------------------------------------------------------------------*/

284: PetscErrorCode PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
285: {
286:   PC_BJacobi   *jac;

289:   jac                 = (PC_BJacobi*)pc->data;
290:   jac->use_true_local = PETSC_TRUE;
291:   return(0);
292: }

298: PetscErrorCode PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
299: {
300:   PC_BJacobi   *jac = (PC_BJacobi*)pc->data;;

303:   if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");

305:   if (n_local)     *n_local     = jac->n_local;
306:   if (first_local) *first_local = jac->first_local;
307:   *ksp                         = jac->ksp;
308:   jac->same_local_solves        = PETSC_FALSE; /* Assume that local solves are now different;
309:                                                   not necessarily true though!  This flag is 
310:                                                   used only for PCView_BJacobi() */
311:   return(0);
312: }

318: PetscErrorCode PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
319: {
320:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;


325:   if (pc->setupcalled > 0) SETERRQ(PETSC_ERR_ORDER,"Cannot set number of blocks after PCSetUp()/KSPSetUp() has been called");
326:   jac->n = blocks;
327:   if (!lens) {
328:     jac->g_lens = 0;
329:   } else {
330:     PetscMalloc(blocks*sizeof(PetscInt),&jac->g_lens);
331:     PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
332:     PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
333:   }
334:   return(0);
335: }

341: PetscErrorCode PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
342: {
343:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

346:   *blocks = jac->n;
347:   if (lens) *lens = jac->g_lens;
348:   return(0);
349: }

355: PetscErrorCode PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
356: {
357:   PC_BJacobi     *jac;

361:   jac = (PC_BJacobi*)pc->data;

363:   jac->n_local = blocks;
364:   if (!lens) {
365:     jac->l_lens = 0;
366:   } else {
367:     PetscMalloc(blocks*sizeof(PetscInt),&jac->l_lens);
368:     PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
369:     PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
370:   }
371:   return(0);
372: }

378: PetscErrorCode PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
379: {
380:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

383:   *blocks = jac->n_local;
384:   if (lens) *lens = jac->l_lens;
385:   return(0);
386: }

389: /* -------------------------------------------------------------------------------------*/

393: /*@
394:    PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block 
395:    problem is associated with the linear system matrix instead of the
396:    default (where it is associated with the preconditioning matrix).
397:    That is, if the local system is solved iteratively then it iterates
398:    on the block from the matrix using the block from the preconditioner
399:    as the preconditioner for the local block.

401:    Collective on PC

403:    Input Parameters:
404: .  pc - the preconditioner context

406:    Options Database Key:
407: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

409:    Notes:
410:    For the common case in which the preconditioning and linear 
411:    system matrices are identical, this routine is unnecessary.

413:    Level: intermediate

415: .keywords:  block, Jacobi, set, true, local, flag

417: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
418: @*/
419: PetscErrorCode PCBJacobiSetUseTrueLocal(PC pc)
420: {
421:   PetscErrorCode ierr,(*f)(PC);

425:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
426:   if (f) {
427:     (*f)(pc);
428:   }

430:   return(0);
431: }

435: /*@C
436:    PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
437:    this processor.
438:    
439:    Note Collective

441:    Input Parameter:
442: .  pc - the preconditioner context

444:    Output Parameters:
445: +  n_local - the number of blocks on this processor, or PETSC_NULL
446: .  first_local - the global number of the first block on this processor, or PETSC_NULL
447: -  ksp - the array of KSP contexts

449:    Notes:  
450:    After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
451:    
452:    Currently for some matrix implementations only 1 block per processor 
453:    is supported.
454:    
455:    You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().

457:    Level: advanced

459: .keywords:  block, Jacobi, get, sub, KSP, context

461: .seealso: PCBJacobiGetSubKSP()
462: @*/
463: PetscErrorCode PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
464: {
465:   PetscErrorCode ierr,(*f)(PC,PetscInt *,PetscInt *,KSP **);

469:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
470:   if (f) {
471:     (*f)(pc,n_local,first_local,ksp);
472:   } else {
473:     SETERRQ(PETSC_ERR_ARG_WRONG,"Cannot get subsolvers for this preconditioner");
474:   }
475:   return(0);
476: }

480: /*@
481:    PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
482:    Jacobi preconditioner.

484:    Collective on PC

486:    Input Parameters:
487: +  pc - the preconditioner context
488: .  blocks - the number of blocks
489: -  lens - [optional] integer array containing the size of each block

491:    Options Database Key:
492: .  -pc_bjacobi_blocks <blocks> - Sets the number of global blocks

494:    Notes:  
495:    Currently only a limited number of blocking configurations are supported.
496:    All processors sharing the PC must call this routine with the same data.

498:    Level: intermediate

500: .keywords:  set, number, Jacobi, global, total, blocks

502: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
503: @*/
504: PetscErrorCode PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
505: {
506:   PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt[]);

510:   if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
511:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
512:   if (f) {
513:     (*f)(pc,blocks,lens);
514:   }
515:   return(0);
516: }

520: /*@C
521:    PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
522:    Jacobi preconditioner.

524:    Collective on PC

526:    Input Parameter:
527: .  pc - the preconditioner context

529:    Output parameters:
530: +  blocks - the number of blocks
531: -  lens - integer array containing the size of each block

533:    Level: intermediate

535: .keywords:  get, number, Jacobi, global, total, blocks

537: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
538: @*/
539: PetscErrorCode PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
540: {
541:   PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);

546:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
547:   if (f) {
548:     (*f)(pc,blocks,lens);
549:   }
550:   return(0);
551: }
552: 
555: /*@
556:    PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
557:    Jacobi preconditioner.

559:    Not Collective

561:    Input Parameters:
562: +  pc - the preconditioner context
563: .  blocks - the number of blocks
564: -  lens - [optional] integer array containing size of each block

566:    Note:  
567:    Currently only a limited number of blocking configurations are supported.

569:    Level: intermediate

571: .keywords: PC, set, number, Jacobi, local, blocks

573: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
574: @*/
575: PetscErrorCode PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
576: {
577:   PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt []);

581:   if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
582:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
583:   if (f) {
584:     (*f)(pc,blocks,lens);
585:   }
586:   return(0);
587: }
588: 
591: /*@C
592:    PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
593:    Jacobi preconditioner.

595:    Not Collective

597:    Input Parameters:
598: +  pc - the preconditioner context
599: .  blocks - the number of blocks
600: -  lens - [optional] integer array containing size of each block

602:    Note:  
603:    Currently only a limited number of blocking configurations are supported.

605:    Level: intermediate

607: .keywords: PC, get, number, Jacobi, local, blocks

609: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
610: @*/
611: PetscErrorCode PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
612: {
613:   PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);

618:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
619:   if (f) {
620:     (*f)(pc,blocks,lens);
621:   }
622:   return(0);
623: }

625: /* -----------------------------------------------------------------------------------*/

627: /*MC
628:    PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with 
629:            its own KSP object.

631:    Options Database Keys:
632: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

634:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
635:      than one processor. Defaults to one block per processor.

637:      To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
638:         options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
639:         
640:      To set the options on the solvers seperate for each block call PCBJacobiGetSubKSP()
641:          and set the options directly on the resulting KSP object (you can access its PC
642:          KSPGetPC())

644:    Level: beginner

646:    Concepts: block Jacobi

648: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
649:            PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
650:            PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
651: M*/

656: PetscErrorCode PCCreate_BJacobi(PC pc)
657: {
659:   PetscMPIInt    rank;
660:   PC_BJacobi     *jac;

663:   PetscNew(PC_BJacobi,&jac);
664:   PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
665:   MPI_Comm_rank(pc->comm,&rank);
666:   pc->ops->apply              = 0;
667:   pc->ops->applytranspose     = 0;
668:   pc->ops->setup              = PCSetUp_BJacobi;
669:   pc->ops->destroy            = PCDestroy_BJacobi;
670:   pc->ops->setfromoptions     = PCSetFromOptions_BJacobi;
671:   pc->ops->view               = PCView_BJacobi;
672:   pc->ops->applyrichardson    = 0;

674:   pc->data               = (void*)jac;
675:   jac->n                 = -1;
676:   jac->n_local           = -1;
677:   jac->first_local       = rank;
678:   jac->ksp              = 0;
679:   jac->use_true_local    = PETSC_FALSE;
680:   jac->same_local_solves = PETSC_TRUE;
681:   jac->g_lens            = 0;
682:   jac->l_lens            = 0;
683:   jac->tp_mat            = 0;
684:   jac->tp_pmat           = 0;

686:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
687:                     "PCBJacobiSetUseTrueLocal_BJacobi",
688:                     PCBJacobiSetUseTrueLocal_BJacobi);
689:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
690:                     PCBJacobiGetSubKSP_BJacobi);
691:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
692:                     PCBJacobiSetTotalBlocks_BJacobi);
693:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
694:                     PCBJacobiGetTotalBlocks_BJacobi);
695:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
696:                     PCBJacobiSetLocalBlocks_BJacobi);
697:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
698:                     PCBJacobiGetLocalBlocks_BJacobi);

700:   return(0);
701: }

704: /* --------------------------------------------------------------------------------------------*/
705: /*
706:         These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
707: */
710: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
711: {
712:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
713:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
714:   PetscErrorCode         ierr;

717:   /*
718:         If the on processor block had to be generated via a MatGetDiagonalBlock()
719:      that creates a copy (for example MPIBDiag matrices do), this frees the space
720:   */
721:   if (jac->tp_mat) {
722:     MatDestroy(jac->tp_mat);
723:   }
724:   if (jac->tp_pmat) {
725:     MatDestroy(jac->tp_pmat);
726:   }

728:   KSPDestroy(jac->ksp[0]);
729:   PetscFree(jac->ksp);
730:   VecDestroy(bjac->x);
731:   VecDestroy(bjac->y);
732:   if (jac->l_lens) {PetscFree(jac->l_lens);}
733:   if (jac->g_lens) {PetscFree(jac->g_lens);}
734:   PetscFree(bjac);
735:   PetscFree(jac);
736:   return(0);
737: }

741: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
742: {
744:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;

747:   KSPSetUp(jac->ksp[0]);
748:   return(0);
749: }

753: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
754: {
755:   PetscErrorCode         ierr;
756:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
757:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
758:   PetscScalar            *x_array,*y_array;

761:   /* 
762:       The VecPlaceArray() is to avoid having to copy the 
763:     y vector into the bjac->x vector. The reason for 
764:     the bjac->x vector is that we need a sequential vector
765:     for the sequential solve.
766:   */
767:   VecGetArray(x,&x_array);
768:   VecGetArray(y,&y_array);
769:   VecPlaceArray(bjac->x,x_array);
770:   VecPlaceArray(bjac->y,y_array);
771:   KSPSolve(jac->ksp[0],bjac->x,bjac->y);
772:   VecRestoreArray(x,&x_array);
773:   VecRestoreArray(y,&y_array);
774:   return(0);
775: }

779: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
780: {
781:   PetscErrorCode         ierr;
782:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
783:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
784:   PetscScalar            *x_array,*y_array;
785:   PC                     subpc;

788:   /* 
789:       The VecPlaceArray() is to avoid having to copy the 
790:     y vector into the bjac->x vector. The reason for 
791:     the bjac->x vector is that we need a sequential vector
792:     for the sequential solve.
793:   */
794:   VecGetArray(x,&x_array);
795:   VecGetArray(y,&y_array);
796:   VecPlaceArray(bjac->x,x_array);
797:   VecPlaceArray(bjac->y,y_array);

799:   /* apply the symmetric left portion of the inner PC operator */
800:   /* note this by-passes the inner KSP and its options completely */

802:   KSPGetPC(jac->ksp[0],&subpc);
803:   PCApplySymmetricLeft(subpc,bjac->x,bjac->y);

805:   VecRestoreArray(x,&x_array);
806:   VecRestoreArray(y,&y_array);
807:   return(0);
808: }

812: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
813: {
814:   PetscErrorCode         ierr;
815:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
816:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
817:   PetscScalar            *x_array,*y_array;
818:   PC                     subpc;

821:   /* 
822:       The VecPlaceArray() is to avoid having to copy the 
823:     y vector into the bjac->x vector. The reason for 
824:     the bjac->x vector is that we need a sequential vector
825:     for the sequential solve.
826:   */
827:   VecGetArray(x,&x_array);
828:   VecGetArray(y,&y_array);
829:   VecPlaceArray(bjac->x,x_array);
830:   VecPlaceArray(bjac->y,y_array);

832:   /* apply the symmetric right portion of the inner PC operator */
833:   /* note this by-passes the inner KSP and its options completely */

835:   KSPGetPC(jac->ksp[0],&subpc);
836:   PCApplySymmetricRight(subpc,bjac->x,bjac->y);

838:   VecRestoreArray(x,&x_array);
839:   VecRestoreArray(y,&y_array);
840:   return(0);
841: }

845: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
846: {
847:   PetscErrorCode         ierr;
848:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
849:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
850:   PetscScalar            *x_array,*y_array;

853:   /* 
854:       The VecPlaceArray() is to avoid having to copy the 
855:     y vector into the bjac->x vector. The reason for 
856:     the bjac->x vector is that we need a sequential vector
857:     for the sequential solve.
858:   */
859:   VecGetArray(x,&x_array);
860:   VecGetArray(y,&y_array);
861:   VecPlaceArray(bjac->x,x_array);
862:   VecPlaceArray(bjac->y,y_array);
863:   KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
864:   VecRestoreArray(x,&x_array);
865:   VecRestoreArray(y,&y_array);
866:   return(0);
867: }

871: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
872: {
873:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
874:   PetscErrorCode         ierr;
875:   PetscInt               m;
876:   KSP                    ksp;
877:   Vec                    x,y;
878:   PC_BJacobi_Singleblock *bjac;
879:   PC                     subpc;


883:   /* set default direct solver with no Krylov method */
884:   if (!pc->setupcalled) {
885:     char *prefix;
886:     KSPCreate(PETSC_COMM_SELF,&ksp);
887:     PetscLogObjectParent(pc,ksp);
888:     KSPSetType(ksp,KSPPREONLY);
889:     KSPGetPC(ksp,&subpc);
890:     PCGetOptionsPrefix(pc,&prefix);
891:     KSPSetOptionsPrefix(ksp,prefix);
892:     KSPAppendOptionsPrefix(ksp,"sub_");
893:     /*
894:       The reason we need to generate these vectors is to serve 
895:       as the right-hand side and solution vector for the solve on the 
896:       block. We do not need to allocate space for the vectors since
897:       that is provided via VecPlaceArray() just before the call to 
898:       KSPSolve() on the block.
899:     */
900:     MatGetSize(pmat,&m,&m);
901:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
902:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
903:     PetscLogObjectParent(pc,x);
904:     PetscLogObjectParent(pc,y);

906:     pc->ops->destroy             = PCDestroy_BJacobi_Singleblock;
907:     pc->ops->apply               = PCApply_BJacobi_Singleblock;
908:     pc->ops->applysymmetricleft  = PCApplySymmetricLeft_BJacobi_Singleblock;
909:     pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
910:     pc->ops->applytranspose      = PCApplyTranspose_BJacobi_Singleblock;
911:     pc->ops->setuponblocks       = PCSetUpOnBlocks_BJacobi_Singleblock;

913:     PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
914:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
915:     bjac->x      = x;
916:     bjac->y      = y;

918:     PetscMalloc(sizeof(KSP),&jac->ksp);
919:     jac->ksp[0] = ksp;
920:     jac->data    = (void*)bjac;
921:   } else {
922:     ksp = jac->ksp[0];
923:     bjac = (PC_BJacobi_Singleblock *)jac->data;
924:   }
925:   if (jac->use_true_local) {
926:     KSPSetOperators(ksp,mat,pmat,pc->flag);
927:   }  else {
928:     KSPSetOperators(ksp,pmat,pmat,pc->flag);
929:   }
930:   KSPSetFromOptions(ksp);
931:   return(0);
932: }

934: /* ---------------------------------------------------------------------------------------------*/

938: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
939: {
940:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
941:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
942:   PetscErrorCode        ierr;
943:   PetscInt              i;

946:   MatDestroyMatrices(jac->n_local,&bjac->pmat);
947:   if (jac->use_true_local) {
948:     MatDestroyMatrices(jac->n_local,&bjac->mat);
949:   }

951:   /*
952:         If the on processor block had to be generated via a MatGetDiagonalBlock()
953:      that creates a copy (for example MPIBDiag matrices do), this frees the space
954:   */
955:   if (jac->tp_mat) {
956:     MatDestroy(jac->tp_mat);
957:   }
958:   if (jac->tp_pmat) {
959:     MatDestroy(jac->tp_pmat);
960:   }

962:   for (i=0; i<jac->n_local; i++) {
963:     KSPDestroy(jac->ksp[i]);
964:     VecDestroy(bjac->x[i]);
965:     VecDestroy(bjac->y[i]);
966:     ISDestroy(bjac->is[i]);
967:   }
968:   PetscFree(jac->ksp);
969:   PetscFree(bjac->x);
970:   PetscFree(bjac->starts);
971:   PetscFree(bjac->is);
972:   PetscFree(bjac);
973:   if (jac->l_lens) {PetscFree(jac->l_lens);}
974:   if (jac->g_lens) {PetscFree(jac->g_lens);}
975:   PetscFree(jac);
976:   return(0);
977: }

981: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
982: {
983:   PC_BJacobi     *jac = (PC_BJacobi*)pc->data;
985:   PetscInt       i,n_local = jac->n_local;

988:   for (i=0; i<n_local; i++) {
989:     KSPSetUp(jac->ksp[i]);
990:   }
991:   return(0);
992: }

994: /*
995:       Preconditioner for block Jacobi 
996: */
999: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1000: {
1001:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
1002:   PetscErrorCode        ierr;
1003:   PetscInt              i,n_local = jac->n_local;
1004:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1005:   PetscScalar           *xin,*yin;
1006:   static PetscTruth     flag = PETSC_TRUE;
1007:   static PetscEvent     SUBKspSolve;

1010:   if (flag) {
1011:     PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1012:     flag = PETSC_FALSE;
1013:   }
1014:   VecGetArray(x,&xin);
1015:   VecGetArray(y,&yin);
1016:   for (i=0; i<n_local; i++) {
1017:     /* 
1018:        To avoid copying the subvector from x into a workspace we instead 
1019:        make the workspace vector array point to the subpart of the array of
1020:        the global vector.
1021:     */
1022:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1023:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

1025:     PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1026:     KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
1027:     PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1028:   }
1029:   VecRestoreArray(x,&xin);
1030:   VecRestoreArray(y,&yin);
1031:   return(0);
1032: }

1034: /*
1035:       Preconditioner for block Jacobi 
1036: */
1039: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1040: {
1041:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
1042:   PetscErrorCode        ierr;
1043:   PetscInt              i,n_local = jac->n_local;
1044:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1045:   PetscScalar           *xin,*yin;
1046:   static PetscTruth     flag = PETSC_TRUE;
1047:   static PetscEvent     SUBKspSolve;

1050:   if (flag) {
1051:     PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1052:     flag = PETSC_FALSE;
1053:   }
1054:   VecGetArray(x,&xin);
1055:   VecGetArray(y,&yin);
1056:   for (i=0; i<n_local; i++) {
1057:     /* 
1058:        To avoid copying the subvector from x into a workspace we instead 
1059:        make the workspace vector array point to the subpart of the array of
1060:        the global vector.
1061:     */
1062:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1063:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

1065:     PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1066:     KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
1067:     PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1068:   }
1069:   VecRestoreArray(x,&xin);
1070:   VecRestoreArray(y,&yin);
1071:   return(0);
1072: }

1076: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1077: {
1078:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
1079:   PetscErrorCode         ierr;
1080:   PetscInt               m,n_local,N,M,start,i;
1081:   char                   *prefix,*pprefix,*mprefix;
1082:   KSP                    ksp;
1083:   Vec                    x,y;
1084:   PC_BJacobi_Multiblock  *bjac = (PC_BJacobi_Multiblock*)jac->data;
1085:   PC                     subpc;
1086:   IS                     is;
1087:   MatReuse               scall = MAT_REUSE_MATRIX;

1090:   MatGetLocalSize(pc->pmat,&M,&N);

1092:   n_local = jac->n_local;

1094:   if (jac->use_true_local) {
1095:     if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1096:   }

1098:   /* set default direct solver with no Krylov method */
1099:   if (!pc->setupcalled) {
1100:     scall                  = MAT_INITIAL_MATRIX;
1101:     pc->ops->destroy       = PCDestroy_BJacobi_Multiblock;
1102:     pc->ops->apply         = PCApply_BJacobi_Multiblock;
1103:     pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1104:     pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;

1106:     PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1107:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1108:     PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1109:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1110:     PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1111:     PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1112:     bjac->y      = bjac->x + n_local;
1113:     PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1114:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1115: 
1116:     jac->data    = (void*)bjac;
1117:     PetscMalloc(n_local*sizeof(IS),&bjac->is);
1118:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));

1120:     start = 0;
1121:     for (i=0; i<n_local; i++) {
1122:       KSPCreate(PETSC_COMM_SELF,&ksp);
1123:       PetscLogObjectParent(pc,ksp);
1124:       KSPSetType(ksp,KSPPREONLY);
1125:       KSPGetPC(ksp,&subpc);
1126:       PCGetOptionsPrefix(pc,&prefix);
1127:       KSPSetOptionsPrefix(ksp,prefix);
1128:       KSPAppendOptionsPrefix(ksp,"sub_");

1130:       m = jac->l_lens[i];

1132:       /*
1133:       The reason we need to generate these vectors is to serve 
1134:       as the right-hand side and solution vector for the solve on the 
1135:       block. We do not need to allocate space for the vectors since
1136:       that is provided via VecPlaceArray() just before the call to 
1137:       KSPSolve() on the block.

1139:       */
1140:       VecCreateSeq(PETSC_COMM_SELF,m,&x);
1141:       VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1142:       PetscLogObjectParent(pc,x);
1143:       PetscLogObjectParent(pc,y);
1144:       bjac->x[i]      = x;
1145:       bjac->y[i]      = y;
1146:       bjac->starts[i] = start;
1147:       jac->ksp[i]    = ksp;

1149:       ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1150:       bjac->is[i] = is;
1151:       PetscLogObjectParent(pc,is);

1153:       start += m;
1154:     }
1155:   } else {
1156:     bjac = (PC_BJacobi_Multiblock*)jac->data;
1157:     /* 
1158:        Destroy the blocks from the previous iteration
1159:     */
1160:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1161:       MatDestroyMatrices(n_local,&bjac->pmat);
1162:       if (jac->use_true_local) {
1163:         MatDestroyMatrices(n_local,&bjac->mat);
1164:       }
1165:       scall = MAT_INITIAL_MATRIX;
1166:     }
1167:   }

1169:   MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1170:   if (jac->use_true_local) {
1171:     PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1172:     MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1173:   }
1174:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
1175:      different boundary conditions for the submatrices than for the global problem) */
1176:   PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);

1178:   PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1179:   for (i=0; i<n_local; i++) {
1180:     PetscLogObjectParent(pc,bjac->pmat[i]);
1181:     PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1182:     if (jac->use_true_local) {
1183:       PetscLogObjectParent(pc,bjac->mat[i]);
1184:       PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1185:       KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1186:     } else {
1187:       KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1188:     }
1189:     KSPSetFromOptions(jac->ksp[i]);
1190:   }

1192:   return(0);
1193: }