Actual source code: bjacobi.c
1: /*
2: Defines a block Jacobi preconditioner.
3: */
4: #include src/mat/matimpl.h
5: #include src/ksp/pc/pcimpl.h
6: #include src/ksp/pc/impls/bjacobi/bjacobi.h
8: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
9: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
13: static PetscErrorCode PCSetUp_BJacobi(PC pc)
14: {
15: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
16: Mat mat = pc->mat,pmat = pc->pmat;
17: PetscErrorCode ierr,(*f)(Mat,PetscTruth*,MatReuse,Mat*);
18: PetscInt N,M,start,i,sum,end;
19: PetscInt bs,i_start=-1,i_end=-1;
20: PetscMPIInt rank,size;
21: char *pprefix,*mprefix;
24: MPI_Comm_rank(pc->comm,&rank);
25: MPI_Comm_size(pc->comm,&size);
26: MatGetLocalSize(pc->pmat,&M,&N);
27: MatGetBlockSize(pc->pmat,&bs);
29: /* ----------
30: Determines the number of blocks assigned to each processor
31: */
33: /* local block count given */
34: if (jac->n_local > 0 && jac->n < 0) {
35: MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,pc->comm);
36: if (jac->l_lens) { /* check that user set these correctly */
37: sum = 0;
38: for (i=0; i<jac->n_local; i++) {
39: if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
40: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
41: }
42: sum += jac->l_lens[i];
43: }
44: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
45: } else {
46: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
47: for (i=0; i<jac->n_local; i++) {
48: jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
49: }
50: }
51: } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
52: /* global blocks given: determine which ones are local */
53: if (jac->g_lens) {
54: /* check if the g_lens is has valid entries */
55: for (i=0; i<jac->n; i++) {
56: if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
57: if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
58: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
59: }
60: }
61: if (size == 1) {
62: jac->n_local = jac->n;
63: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
64: PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
65: /* check that user set these correctly */
66: sum = 0;
67: for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
68: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
69: } else {
70: MatGetOwnershipRange(pc->pmat,&start,&end);
71: /* loop over blocks determing first one owned by me */
72: sum = 0;
73: for (i=0; i<jac->n+1; i++) {
74: if (sum == start) { i_start = i; goto start_1;}
75: if (i < jac->n) sum += jac->g_lens[i];
76: }
77: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
78: used in PCBJacobiSetTotalBlocks()\n\
79: are not compatible with parallel matrix layout");
80: start_1:
81: for (i=i_start; i<jac->n+1; i++) {
82: if (sum == end) { i_end = i; goto end_1; }
83: if (i < jac->n) sum += jac->g_lens[i];
84: }
85: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
86: used in PCBJacobiSetTotalBlocks()\n\
87: are not compatible with parallel matrix layout");
88: end_1:
89: jac->n_local = i_end - i_start;
90: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
91: PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
92: }
93: } else { /* no global blocks given, determine then using default layout */
94: jac->n_local = jac->n/size + ((jac->n % size) > rank);
95: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
96: for (i=0; i<jac->n_local; i++) {
97: jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
98: if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
99: }
100: }
101: } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
102: jac->n = size;
103: jac->n_local = 1;
104: PetscMalloc(sizeof(PetscInt),&jac->l_lens);
105: jac->l_lens[0] = M;
106: }
108: MPI_Comm_size(pc->comm,&size);
109: PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
110: if (size == 1 && !f) {
111: mat = pc->mat;
112: pmat = pc->pmat;
113: } else {
114: PetscTruth iscopy;
115: MatReuse scall;
117: if (jac->use_true_local) {
118: scall = MAT_INITIAL_MATRIX;
119: if (pc->setupcalled) {
120: if (pc->flag == SAME_NONZERO_PATTERN) {
121: if (jac->tp_mat) {
122: scall = MAT_REUSE_MATRIX;
123: mat = jac->tp_mat;
124: }
125: } else {
126: if (jac->tp_mat) {
127: MatDestroy(jac->tp_mat);
128: }
129: }
130: }
131: if (!f) {
132: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
133: }
134: (*f)(pc->mat,&iscopy,scall,&mat);
135: /* make submatrix have same prefix as entire matrix */
136: PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
137: PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
138: if (iscopy) {
139: jac->tp_mat = mat;
140: }
141: }
142: if (pc->pmat != pc->mat || !jac->use_true_local) {
143: scall = MAT_INITIAL_MATRIX;
144: if (pc->setupcalled) {
145: if (pc->flag == SAME_NONZERO_PATTERN) {
146: if (jac->tp_pmat) {
147: scall = MAT_REUSE_MATRIX;
148: pmat = jac->tp_pmat;
149: }
150: } else {
151: if (jac->tp_pmat) {
152: MatDestroy(jac->tp_pmat);
153: }
154: }
155: }
156: PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
157: if (!f) {
158: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
159: }
160: (*f)(pc->pmat,&iscopy,scall,&pmat);
161: /* make submatrix have same prefix as entire matrix */
162: PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
163: PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
164: if (iscopy) {
165: jac->tp_pmat = pmat;
166: }
167: } else {
168: pmat = mat;
169: }
170: }
172: /* ------
173: Setup code depends on the number of blocks
174: */
175: if (jac->n_local == 1) {
176: PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
177: } else {
178: PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
179: }
180: return(0);
181: }
183: /* Default destroy, if it has never been setup */
186: static PetscErrorCode PCDestroy_BJacobi(PC pc)
187: {
188: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
192: if (jac->g_lens) {PetscFree(jac->g_lens);}
193: if (jac->l_lens) {PetscFree(jac->l_lens);}
194: PetscFree(jac);
195: return(0);
196: }
200: static PetscErrorCode PCSetFromOptions_BJacobi(PC pc)
201: {
202: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
204: PetscInt blocks;
205: PetscTruth flg;
208: PetscOptionsHead("Block Jacobi options");
209: PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
210: if (flg) {
211: PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
212: }
213: PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
214: if (flg) {
215: PCBJacobiSetUseTrueLocal(pc);
216: }
217: PetscOptionsTail();
218: return(0);
219: }
223: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
224: {
225: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
227: PetscMPIInt rank;
228: PetscInt i;
229: PetscTruth iascii,isstring;
230: PetscViewer sviewer;
233: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
234: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
235: if (iascii) {
236: if (jac->use_true_local) {
237: PetscViewerASCIIPrintf(viewer," block Jacobi: using true local matrix, number of blocks = %D\n",jac->n);
238: }
239: PetscViewerASCIIPrintf(viewer," block Jacobi: number of blocks = %D\n",jac->n);
240: MPI_Comm_rank(pc->comm,&rank);
241: if (jac->same_local_solves) {
242: PetscViewerASCIIPrintf(viewer," Local solve is same for all blocks, in the following KSP and PC objects:\n");
243: PetscViewerGetSingleton(viewer,&sviewer);
244: if (!rank && jac->ksp) {
245: PetscViewerASCIIPushTab(viewer);
246: KSPView(jac->ksp[0],sviewer);
247: PetscViewerASCIIPopTab(viewer);
248: }
249: PetscViewerRestoreSingleton(viewer,&sviewer);
250: } else {
252: PetscViewerASCIIPrintf(viewer," Local solve info for each block is in the following KSP and PC objects:\n");
253: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
254: rank,jac->n_local,jac->first_local);
255: PetscViewerASCIIPushTab(viewer);
256: for (i=0; i<jac->n_local; i++) {
257: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
258: PetscViewerGetSingleton(viewer,&sviewer);
259: KSPView(jac->ksp[i],sviewer);
260: if (i != jac->n_local-1) {
261: PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
262: }
263: PetscViewerRestoreSingleton(viewer,&sviewer);
264: }
265: PetscViewerASCIIPopTab(viewer);
266: PetscViewerFlush(viewer);
267: }
268: } else if (isstring) {
269: PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
270: PetscViewerGetSingleton(viewer,&sviewer);
271: if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
272: PetscViewerRestoreSingleton(viewer,&sviewer);
273: } else {
274: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
275: }
276: return(0);
277: }
279: /* -------------------------------------------------------------------------------------*/
284: PetscErrorCode PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
285: {
286: PC_BJacobi *jac;
289: jac = (PC_BJacobi*)pc->data;
290: jac->use_true_local = PETSC_TRUE;
291: return(0);
292: }
298: PetscErrorCode PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
299: {
300: PC_BJacobi *jac = (PC_BJacobi*)pc->data;;
303: if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");
305: if (n_local) *n_local = jac->n_local;
306: if (first_local) *first_local = jac->first_local;
307: *ksp = jac->ksp;
308: jac->same_local_solves = PETSC_FALSE; /* Assume that local solves are now different;
309: not necessarily true though! This flag is
310: used only for PCView_BJacobi() */
311: return(0);
312: }
318: PetscErrorCode PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
319: {
320: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
325: if (pc->setupcalled > 0) SETERRQ(PETSC_ERR_ORDER,"Cannot set number of blocks after PCSetUp()/KSPSetUp() has been called");
326: jac->n = blocks;
327: if (!lens) {
328: jac->g_lens = 0;
329: } else {
330: PetscMalloc(blocks*sizeof(PetscInt),&jac->g_lens);
331: PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
332: PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
333: }
334: return(0);
335: }
341: PetscErrorCode PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
342: {
343: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
346: *blocks = jac->n;
347: if (lens) *lens = jac->g_lens;
348: return(0);
349: }
355: PetscErrorCode PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
356: {
357: PC_BJacobi *jac;
361: jac = (PC_BJacobi*)pc->data;
363: jac->n_local = blocks;
364: if (!lens) {
365: jac->l_lens = 0;
366: } else {
367: PetscMalloc(blocks*sizeof(PetscInt),&jac->l_lens);
368: PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
369: PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
370: }
371: return(0);
372: }
378: PetscErrorCode PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
379: {
380: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
383: *blocks = jac->n_local;
384: if (lens) *lens = jac->l_lens;
385: return(0);
386: }
389: /* -------------------------------------------------------------------------------------*/
393: /*@
394: PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block
395: problem is associated with the linear system matrix instead of the
396: default (where it is associated with the preconditioning matrix).
397: That is, if the local system is solved iteratively then it iterates
398: on the block from the matrix using the block from the preconditioner
399: as the preconditioner for the local block.
401: Collective on PC
403: Input Parameters:
404: . pc - the preconditioner context
406: Options Database Key:
407: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
409: Notes:
410: For the common case in which the preconditioning and linear
411: system matrices are identical, this routine is unnecessary.
413: Level: intermediate
415: .keywords: block, Jacobi, set, true, local, flag
417: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
418: @*/
419: PetscErrorCode PCBJacobiSetUseTrueLocal(PC pc)
420: {
421: PetscErrorCode ierr,(*f)(PC);
425: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
426: if (f) {
427: (*f)(pc);
428: }
430: return(0);
431: }
435: /*@C
436: PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
437: this processor.
438:
439: Note Collective
441: Input Parameter:
442: . pc - the preconditioner context
444: Output Parameters:
445: + n_local - the number of blocks on this processor, or PETSC_NULL
446: . first_local - the global number of the first block on this processor, or PETSC_NULL
447: - ksp - the array of KSP contexts
449: Notes:
450: After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
451:
452: Currently for some matrix implementations only 1 block per processor
453: is supported.
454:
455: You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().
457: Level: advanced
459: .keywords: block, Jacobi, get, sub, KSP, context
461: .seealso: PCBJacobiGetSubKSP()
462: @*/
463: PetscErrorCode PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
464: {
465: PetscErrorCode ierr,(*f)(PC,PetscInt *,PetscInt *,KSP **);
469: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
470: if (f) {
471: (*f)(pc,n_local,first_local,ksp);
472: } else {
473: SETERRQ(PETSC_ERR_ARG_WRONG,"Cannot get subsolvers for this preconditioner");
474: }
475: return(0);
476: }
480: /*@
481: PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
482: Jacobi preconditioner.
484: Collective on PC
486: Input Parameters:
487: + pc - the preconditioner context
488: . blocks - the number of blocks
489: - lens - [optional] integer array containing the size of each block
491: Options Database Key:
492: . -pc_bjacobi_blocks <blocks> - Sets the number of global blocks
494: Notes:
495: Currently only a limited number of blocking configurations are supported.
496: All processors sharing the PC must call this routine with the same data.
498: Level: intermediate
500: .keywords: set, number, Jacobi, global, total, blocks
502: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
503: @*/
504: PetscErrorCode PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
505: {
506: PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt[]);
510: if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
511: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
512: if (f) {
513: (*f)(pc,blocks,lens);
514: }
515: return(0);
516: }
520: /*@C
521: PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
522: Jacobi preconditioner.
524: Collective on PC
526: Input Parameter:
527: . pc - the preconditioner context
529: Output parameters:
530: + blocks - the number of blocks
531: - lens - integer array containing the size of each block
533: Level: intermediate
535: .keywords: get, number, Jacobi, global, total, blocks
537: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
538: @*/
539: PetscErrorCode PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
540: {
541: PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);
546: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
547: if (f) {
548: (*f)(pc,blocks,lens);
549: }
550: return(0);
551: }
552:
555: /*@
556: PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
557: Jacobi preconditioner.
559: Not Collective
561: Input Parameters:
562: + pc - the preconditioner context
563: . blocks - the number of blocks
564: - lens - [optional] integer array containing size of each block
566: Note:
567: Currently only a limited number of blocking configurations are supported.
569: Level: intermediate
571: .keywords: PC, set, number, Jacobi, local, blocks
573: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
574: @*/
575: PetscErrorCode PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
576: {
577: PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt []);
581: if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
582: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
583: if (f) {
584: (*f)(pc,blocks,lens);
585: }
586: return(0);
587: }
588:
591: /*@C
592: PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
593: Jacobi preconditioner.
595: Not Collective
597: Input Parameters:
598: + pc - the preconditioner context
599: . blocks - the number of blocks
600: - lens - [optional] integer array containing size of each block
602: Note:
603: Currently only a limited number of blocking configurations are supported.
605: Level: intermediate
607: .keywords: PC, get, number, Jacobi, local, blocks
609: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
610: @*/
611: PetscErrorCode PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
612: {
613: PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);
618: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
619: if (f) {
620: (*f)(pc,blocks,lens);
621: }
622: return(0);
623: }
625: /* -----------------------------------------------------------------------------------*/
627: /*MC
628: PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with
629: its own KSP object.
631: Options Database Keys:
632: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
634: Notes: Each processor can have one or more blocks, but a block cannot be shared by more
635: than one processor. Defaults to one block per processor.
637: To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
638: options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
639:
640: To set the options on the solvers seperate for each block call PCBJacobiGetSubKSP()
641: and set the options directly on the resulting KSP object (you can access its PC
642: KSPGetPC())
644: Level: beginner
646: Concepts: block Jacobi
648: .seealso: PCCreate(), PCSetType(), PCType (for list of available types), PC,
649: PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
650: PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
651: M*/
656: PetscErrorCode PCCreate_BJacobi(PC pc)
657: {
659: PetscMPIInt rank;
660: PC_BJacobi *jac;
663: PetscNew(PC_BJacobi,&jac);
664: PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
665: MPI_Comm_rank(pc->comm,&rank);
666: pc->ops->apply = 0;
667: pc->ops->applytranspose = 0;
668: pc->ops->setup = PCSetUp_BJacobi;
669: pc->ops->destroy = PCDestroy_BJacobi;
670: pc->ops->setfromoptions = PCSetFromOptions_BJacobi;
671: pc->ops->view = PCView_BJacobi;
672: pc->ops->applyrichardson = 0;
674: pc->data = (void*)jac;
675: jac->n = -1;
676: jac->n_local = -1;
677: jac->first_local = rank;
678: jac->ksp = 0;
679: jac->use_true_local = PETSC_FALSE;
680: jac->same_local_solves = PETSC_TRUE;
681: jac->g_lens = 0;
682: jac->l_lens = 0;
683: jac->tp_mat = 0;
684: jac->tp_pmat = 0;
686: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
687: "PCBJacobiSetUseTrueLocal_BJacobi",
688: PCBJacobiSetUseTrueLocal_BJacobi);
689: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
690: PCBJacobiGetSubKSP_BJacobi);
691: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
692: PCBJacobiSetTotalBlocks_BJacobi);
693: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
694: PCBJacobiGetTotalBlocks_BJacobi);
695: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
696: PCBJacobiSetLocalBlocks_BJacobi);
697: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
698: PCBJacobiGetLocalBlocks_BJacobi);
700: return(0);
701: }
704: /* --------------------------------------------------------------------------------------------*/
705: /*
706: These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
707: */
710: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
711: {
712: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
713: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
714: PetscErrorCode ierr;
717: /*
718: If the on processor block had to be generated via a MatGetDiagonalBlock()
719: that creates a copy (for example MPIBDiag matrices do), this frees the space
720: */
721: if (jac->tp_mat) {
722: MatDestroy(jac->tp_mat);
723: }
724: if (jac->tp_pmat) {
725: MatDestroy(jac->tp_pmat);
726: }
728: KSPDestroy(jac->ksp[0]);
729: PetscFree(jac->ksp);
730: VecDestroy(bjac->x);
731: VecDestroy(bjac->y);
732: if (jac->l_lens) {PetscFree(jac->l_lens);}
733: if (jac->g_lens) {PetscFree(jac->g_lens);}
734: PetscFree(bjac);
735: PetscFree(jac);
736: return(0);
737: }
741: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
742: {
744: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
747: KSPSetUp(jac->ksp[0]);
748: return(0);
749: }
753: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
754: {
755: PetscErrorCode ierr;
756: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
757: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
758: PetscScalar *x_array,*y_array;
761: /*
762: The VecPlaceArray() is to avoid having to copy the
763: y vector into the bjac->x vector. The reason for
764: the bjac->x vector is that we need a sequential vector
765: for the sequential solve.
766: */
767: VecGetArray(x,&x_array);
768: VecGetArray(y,&y_array);
769: VecPlaceArray(bjac->x,x_array);
770: VecPlaceArray(bjac->y,y_array);
771: KSPSolve(jac->ksp[0],bjac->x,bjac->y);
772: VecRestoreArray(x,&x_array);
773: VecRestoreArray(y,&y_array);
774: return(0);
775: }
779: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
780: {
781: PetscErrorCode ierr;
782: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
783: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
784: PetscScalar *x_array,*y_array;
785: PC subpc;
788: /*
789: The VecPlaceArray() is to avoid having to copy the
790: y vector into the bjac->x vector. The reason for
791: the bjac->x vector is that we need a sequential vector
792: for the sequential solve.
793: */
794: VecGetArray(x,&x_array);
795: VecGetArray(y,&y_array);
796: VecPlaceArray(bjac->x,x_array);
797: VecPlaceArray(bjac->y,y_array);
799: /* apply the symmetric left portion of the inner PC operator */
800: /* note this by-passes the inner KSP and its options completely */
802: KSPGetPC(jac->ksp[0],&subpc);
803: PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
805: VecRestoreArray(x,&x_array);
806: VecRestoreArray(y,&y_array);
807: return(0);
808: }
812: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
813: {
814: PetscErrorCode ierr;
815: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
816: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
817: PetscScalar *x_array,*y_array;
818: PC subpc;
821: /*
822: The VecPlaceArray() is to avoid having to copy the
823: y vector into the bjac->x vector. The reason for
824: the bjac->x vector is that we need a sequential vector
825: for the sequential solve.
826: */
827: VecGetArray(x,&x_array);
828: VecGetArray(y,&y_array);
829: VecPlaceArray(bjac->x,x_array);
830: VecPlaceArray(bjac->y,y_array);
832: /* apply the symmetric right portion of the inner PC operator */
833: /* note this by-passes the inner KSP and its options completely */
835: KSPGetPC(jac->ksp[0],&subpc);
836: PCApplySymmetricRight(subpc,bjac->x,bjac->y);
838: VecRestoreArray(x,&x_array);
839: VecRestoreArray(y,&y_array);
840: return(0);
841: }
845: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
846: {
847: PetscErrorCode ierr;
848: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
849: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
850: PetscScalar *x_array,*y_array;
853: /*
854: The VecPlaceArray() is to avoid having to copy the
855: y vector into the bjac->x vector. The reason for
856: the bjac->x vector is that we need a sequential vector
857: for the sequential solve.
858: */
859: VecGetArray(x,&x_array);
860: VecGetArray(y,&y_array);
861: VecPlaceArray(bjac->x,x_array);
862: VecPlaceArray(bjac->y,y_array);
863: KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
864: VecRestoreArray(x,&x_array);
865: VecRestoreArray(y,&y_array);
866: return(0);
867: }
871: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
872: {
873: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
874: PetscErrorCode ierr;
875: PetscInt m;
876: KSP ksp;
877: Vec x,y;
878: PC_BJacobi_Singleblock *bjac;
879: PC subpc;
883: /* set default direct solver with no Krylov method */
884: if (!pc->setupcalled) {
885: char *prefix;
886: KSPCreate(PETSC_COMM_SELF,&ksp);
887: PetscLogObjectParent(pc,ksp);
888: KSPSetType(ksp,KSPPREONLY);
889: KSPGetPC(ksp,&subpc);
890: PCGetOptionsPrefix(pc,&prefix);
891: KSPSetOptionsPrefix(ksp,prefix);
892: KSPAppendOptionsPrefix(ksp,"sub_");
893: /*
894: The reason we need to generate these vectors is to serve
895: as the right-hand side and solution vector for the solve on the
896: block. We do not need to allocate space for the vectors since
897: that is provided via VecPlaceArray() just before the call to
898: KSPSolve() on the block.
899: */
900: MatGetSize(pmat,&m,&m);
901: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
902: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
903: PetscLogObjectParent(pc,x);
904: PetscLogObjectParent(pc,y);
906: pc->ops->destroy = PCDestroy_BJacobi_Singleblock;
907: pc->ops->apply = PCApply_BJacobi_Singleblock;
908: pc->ops->applysymmetricleft = PCApplySymmetricLeft_BJacobi_Singleblock;
909: pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
910: pc->ops->applytranspose = PCApplyTranspose_BJacobi_Singleblock;
911: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Singleblock;
913: PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
914: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
915: bjac->x = x;
916: bjac->y = y;
918: PetscMalloc(sizeof(KSP),&jac->ksp);
919: jac->ksp[0] = ksp;
920: jac->data = (void*)bjac;
921: } else {
922: ksp = jac->ksp[0];
923: bjac = (PC_BJacobi_Singleblock *)jac->data;
924: }
925: if (jac->use_true_local) {
926: KSPSetOperators(ksp,mat,pmat,pc->flag);
927: } else {
928: KSPSetOperators(ksp,pmat,pmat,pc->flag);
929: }
930: KSPSetFromOptions(ksp);
931: return(0);
932: }
934: /* ---------------------------------------------------------------------------------------------*/
938: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
939: {
940: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
941: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
942: PetscErrorCode ierr;
943: PetscInt i;
946: MatDestroyMatrices(jac->n_local,&bjac->pmat);
947: if (jac->use_true_local) {
948: MatDestroyMatrices(jac->n_local,&bjac->mat);
949: }
951: /*
952: If the on processor block had to be generated via a MatGetDiagonalBlock()
953: that creates a copy (for example MPIBDiag matrices do), this frees the space
954: */
955: if (jac->tp_mat) {
956: MatDestroy(jac->tp_mat);
957: }
958: if (jac->tp_pmat) {
959: MatDestroy(jac->tp_pmat);
960: }
962: for (i=0; i<jac->n_local; i++) {
963: KSPDestroy(jac->ksp[i]);
964: VecDestroy(bjac->x[i]);
965: VecDestroy(bjac->y[i]);
966: ISDestroy(bjac->is[i]);
967: }
968: PetscFree(jac->ksp);
969: PetscFree(bjac->x);
970: PetscFree(bjac->starts);
971: PetscFree(bjac->is);
972: PetscFree(bjac);
973: if (jac->l_lens) {PetscFree(jac->l_lens);}
974: if (jac->g_lens) {PetscFree(jac->g_lens);}
975: PetscFree(jac);
976: return(0);
977: }
981: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
982: {
983: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
985: PetscInt i,n_local = jac->n_local;
988: for (i=0; i<n_local; i++) {
989: KSPSetUp(jac->ksp[i]);
990: }
991: return(0);
992: }
994: /*
995: Preconditioner for block Jacobi
996: */
999: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1000: {
1001: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1002: PetscErrorCode ierr;
1003: PetscInt i,n_local = jac->n_local;
1004: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1005: PetscScalar *xin,*yin;
1006: static PetscTruth flag = PETSC_TRUE;
1007: static PetscEvent SUBKspSolve;
1010: if (flag) {
1011: PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1012: flag = PETSC_FALSE;
1013: }
1014: VecGetArray(x,&xin);
1015: VecGetArray(y,&yin);
1016: for (i=0; i<n_local; i++) {
1017: /*
1018: To avoid copying the subvector from x into a workspace we instead
1019: make the workspace vector array point to the subpart of the array of
1020: the global vector.
1021: */
1022: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1023: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1025: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1026: KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
1027: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1028: }
1029: VecRestoreArray(x,&xin);
1030: VecRestoreArray(y,&yin);
1031: return(0);
1032: }
1034: /*
1035: Preconditioner for block Jacobi
1036: */
1039: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1040: {
1041: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1042: PetscErrorCode ierr;
1043: PetscInt i,n_local = jac->n_local;
1044: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1045: PetscScalar *xin,*yin;
1046: static PetscTruth flag = PETSC_TRUE;
1047: static PetscEvent SUBKspSolve;
1050: if (flag) {
1051: PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1052: flag = PETSC_FALSE;
1053: }
1054: VecGetArray(x,&xin);
1055: VecGetArray(y,&yin);
1056: for (i=0; i<n_local; i++) {
1057: /*
1058: To avoid copying the subvector from x into a workspace we instead
1059: make the workspace vector array point to the subpart of the array of
1060: the global vector.
1061: */
1062: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1063: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1065: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1066: KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
1067: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1068: }
1069: VecRestoreArray(x,&xin);
1070: VecRestoreArray(y,&yin);
1071: return(0);
1072: }
1076: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1077: {
1078: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1079: PetscErrorCode ierr;
1080: PetscInt m,n_local,N,M,start,i;
1081: char *prefix,*pprefix,*mprefix;
1082: KSP ksp;
1083: Vec x,y;
1084: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1085: PC subpc;
1086: IS is;
1087: MatReuse scall = MAT_REUSE_MATRIX;
1090: MatGetLocalSize(pc->pmat,&M,&N);
1092: n_local = jac->n_local;
1094: if (jac->use_true_local) {
1095: if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1096: }
1098: /* set default direct solver with no Krylov method */
1099: if (!pc->setupcalled) {
1100: scall = MAT_INITIAL_MATRIX;
1101: pc->ops->destroy = PCDestroy_BJacobi_Multiblock;
1102: pc->ops->apply = PCApply_BJacobi_Multiblock;
1103: pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1104: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;
1106: PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1107: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1108: PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1109: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1110: PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1111: PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1112: bjac->y = bjac->x + n_local;
1113: PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1114: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1115:
1116: jac->data = (void*)bjac;
1117: PetscMalloc(n_local*sizeof(IS),&bjac->is);
1118: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));
1120: start = 0;
1121: for (i=0; i<n_local; i++) {
1122: KSPCreate(PETSC_COMM_SELF,&ksp);
1123: PetscLogObjectParent(pc,ksp);
1124: KSPSetType(ksp,KSPPREONLY);
1125: KSPGetPC(ksp,&subpc);
1126: PCGetOptionsPrefix(pc,&prefix);
1127: KSPSetOptionsPrefix(ksp,prefix);
1128: KSPAppendOptionsPrefix(ksp,"sub_");
1130: m = jac->l_lens[i];
1132: /*
1133: The reason we need to generate these vectors is to serve
1134: as the right-hand side and solution vector for the solve on the
1135: block. We do not need to allocate space for the vectors since
1136: that is provided via VecPlaceArray() just before the call to
1137: KSPSolve() on the block.
1139: */
1140: VecCreateSeq(PETSC_COMM_SELF,m,&x);
1141: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1142: PetscLogObjectParent(pc,x);
1143: PetscLogObjectParent(pc,y);
1144: bjac->x[i] = x;
1145: bjac->y[i] = y;
1146: bjac->starts[i] = start;
1147: jac->ksp[i] = ksp;
1149: ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1150: bjac->is[i] = is;
1151: PetscLogObjectParent(pc,is);
1153: start += m;
1154: }
1155: } else {
1156: bjac = (PC_BJacobi_Multiblock*)jac->data;
1157: /*
1158: Destroy the blocks from the previous iteration
1159: */
1160: if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1161: MatDestroyMatrices(n_local,&bjac->pmat);
1162: if (jac->use_true_local) {
1163: MatDestroyMatrices(n_local,&bjac->mat);
1164: }
1165: scall = MAT_INITIAL_MATRIX;
1166: }
1167: }
1169: MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1170: if (jac->use_true_local) {
1171: PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1172: MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1173: }
1174: /* Return control to the user so that the submatrices can be modified (e.g., to apply
1175: different boundary conditions for the submatrices than for the global problem) */
1176: PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);
1178: PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1179: for (i=0; i<n_local; i++) {
1180: PetscLogObjectParent(pc,bjac->pmat[i]);
1181: PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1182: if (jac->use_true_local) {
1183: PetscLogObjectParent(pc,bjac->mat[i]);
1184: PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1185: KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1186: } else {
1187: KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1188: }
1189: KSPSetFromOptions(jac->ksp[i]);
1190: }
1192: return(0);
1193: }