Actual source code: gamg.c
petsc-dev 2014-02-02
1: /*
2: GAMG geometric-algebric multigrid PC - Mark Adams 2011
3: */
4: #include <petsc-private/matimpl.h>
5: #include <../src/ksp/pc/impls/gamg/gamg.h> /*I "petscpc.h" I*/
6: #include <petsc-private/kspimpl.h>
7: #include <../src/ksp/pc/impls/bjacobi/bjacobi.h> /* Hack to access same_local_solves */
9: #if defined PETSC_GAMG_USE_LOG
10: PetscLogEvent petsc_gamg_setup_events[NUM_SET];
11: #endif
13: #if defined PETSC_USE_LOG
14: PetscLogEvent PC_GAMGGgraph_AGG;
15: PetscLogEvent PC_GAMGGgraph_GEO;
16: PetscLogEvent PC_GAMGCoarsen_AGG;
17: PetscLogEvent PC_GAMGCoarsen_GEO;
18: PetscLogEvent PC_GAMGProlongator_AGG;
19: PetscLogEvent PC_GAMGProlongator_GEO;
20: PetscLogEvent PC_GAMGOptprol_AGG;
21: #endif
23: #define GAMG_MAXLEVELS 30
25: /* #define GAMG_STAGES */
26: #if (defined PETSC_GAMG_USE_LOG && defined GAMG_STAGES)
27: static PetscLogStage gamg_stages[GAMG_MAXLEVELS];
28: #endif
30: static PetscFunctionList GAMGList = 0;
31: static PetscBool PCGAMGPackageInitialized;
33: /* ----------------------------------------------------------------------------- */
36: PetscErrorCode PCReset_GAMG(PC pc)
37: {
39: PC_MG *mg = (PC_MG*)pc->data;
40: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
43: if (pc_gamg->data) { /* this should not happen, cleaned up in SetUp */
44: PetscPrintf(PetscObjectComm((PetscObject)pc),"***[%d]%s this should not happen, cleaned up in SetUp\n",0,__FUNCT__);
45: PetscFree(pc_gamg->data);
46: }
47: pc_gamg->data = NULL; pc_gamg->data_sz = 0;
49: if (pc_gamg->orig_data) {
50: PetscFree(pc_gamg->orig_data);
51: }
52: return(0);
53: }
55: /* -------------------------------------------------------------------------- */
56: /*
57: createLevel: create coarse op with RAP. repartition and/or reduce number
58: of active processors.
60: Input Parameter:
61: . pc - parameters + side effect: coarse data in 'pc_gamg->data' and
62: 'pc_gamg->data_sz' are changed via repartitioning/reduction.
63: . Amat_fine - matrix on this fine (k) level
64: . cr_bs - coarse block size
65: . isLast -
66: In/Output Parameter:
67: . a_P_inout - prolongation operator to the next level (k-->k-1)
68: . a_nactive_proc - number of active procs
69: Output Parameter:
70: . a_Amat_crs - coarse matrix that is created (k-1)
71: */
75: static PetscErrorCode createLevel(const PC pc,const Mat Amat_fine,const PetscInt cr_bs,const PetscBool isLast,
76: Mat *a_P_inout,Mat *a_Amat_crs,PetscMPIInt *a_nactive_proc)
77: {
78: PetscErrorCode ierr;
79: PC_MG *mg = (PC_MG*)pc->data;
80: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
81: const PetscBool repart = pc_gamg->repart;
82: const PetscInt min_eq_proc = pc_gamg->min_eq_proc, coarse_max = pc_gamg->coarse_eq_limit;
83: Mat Cmat,Pold=*a_P_inout;
84: MPI_Comm comm;
85: PetscMPIInt rank,size,new_size,nactive=*a_nactive_proc;
86: PetscInt ncrs_eq,ncrs,f_bs;
89: PetscObjectGetComm((PetscObject)Amat_fine,&comm);
90: MPI_Comm_rank(comm, &rank);
91: MPI_Comm_size(comm, &size);
92: MatGetBlockSize(Amat_fine, &f_bs);
93: /* RAP */
94: MatPtAP(Amat_fine, Pold, MAT_INITIAL_MATRIX, 2.0, &Cmat);
96: /* set 'ncrs' (nodes), 'ncrs_eq' (equations)*/
97: MatGetLocalSize(Cmat, &ncrs_eq, NULL);
98: if (pc_gamg->data_cell_rows>0) {
99: ncrs = pc_gamg->data_sz/pc_gamg->data_cell_cols/pc_gamg->data_cell_rows;
100: }
101: else {
102: PetscInt bs;
103: MatGetBlockSize(Cmat, &bs);
104: ncrs = ncrs_eq/bs;
105: }
107: /* get number of PEs to make active 'new_size', reduce, can be any integer 1-P */
108: {
109: PetscInt ncrs_eq_glob;
110: MatGetSize(Cmat, &ncrs_eq_glob, NULL);
111: new_size = (PetscMPIInt)((float)ncrs_eq_glob/(float)min_eq_proc + 0.5); /* hardwire min. number of eq/proc */
112: if (new_size == 0 || ncrs_eq_glob < coarse_max) new_size = 1;
113: else if (new_size >= nactive) new_size = nactive; /* no change, rare */
114: if (isLast) new_size = 1;
115: }
117: if (!repart && new_size==nactive) *a_Amat_crs = Cmat; /* output - no repartitioning or reduction - could bail here */
118: else {
119: PetscInt *counts,*newproc_idx,ii,jj,kk,strideNew,*tidx,ncrs_new,ncrs_eq_new,nloc_old;
120: IS is_eq_newproc,is_eq_num,is_eq_num_prim,new_eq_indices;
122: nloc_old = ncrs_eq/cr_bs;
123: if (ncrs_eq % cr_bs) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"ncrs_eq %D not divisible by cr_bs %D",ncrs_eq,cr_bs);
124: #if defined PETSC_GAMG_USE_LOG
125: PetscLogEventBegin(petsc_gamg_setup_events[SET12],0,0,0,0);
126: #endif
127: /* make 'is_eq_newproc' */
128: PetscMalloc1(size, &counts);
129: if (repart) {
130: /* Repartition Cmat_{k} and move colums of P^{k}_{k-1} and coordinates of primal part accordingly */
131: Mat adj;
133: if (pc_gamg->verbose>0) {
134: if (pc_gamg->verbose==1) PetscPrintf(comm,"\t[%d]%s repartition: size (active): %d --> %d, neq = %d\n",rank,__FUNCT__,*a_nactive_proc,new_size,ncrs_eq);
135: else {
136: PetscInt n;
137: MPI_Allreduce(&ncrs_eq, &n, 1, MPIU_INT, MPI_SUM, comm);
138: PetscPrintf(comm,"\t[%d]%s repartition: size (active): %d --> %d, neq = %d\n",rank,__FUNCT__,*a_nactive_proc,new_size,n);
139: }
140: }
142: /* get 'adj' */
143: if (cr_bs == 1) {
144: MatConvert(Cmat, MATMPIADJ, MAT_INITIAL_MATRIX, &adj);
145: } else {
146: /* make a scalar matrix to partition (no Stokes here) */
147: Mat tMat;
148: PetscInt Istart_crs,Iend_crs,ncols,jj,Ii;
149: const PetscScalar *vals;
150: const PetscInt *idx;
151: PetscInt *d_nnz, *o_nnz, M, N;
152: static PetscInt llev = 0;
154: PetscMalloc1(ncrs, &d_nnz);
155: PetscMalloc1(ncrs, &o_nnz);
156: MatGetOwnershipRange(Cmat, &Istart_crs, &Iend_crs);
157: MatGetSize(Cmat, &M, &N);
158: for (Ii = Istart_crs, jj = 0; Ii < Iend_crs; Ii += cr_bs, jj++) {
159: MatGetRow(Cmat,Ii,&ncols,0,0);
160: d_nnz[jj] = ncols/cr_bs;
161: o_nnz[jj] = ncols/cr_bs;
162: MatRestoreRow(Cmat,Ii,&ncols,0,0);
163: if (d_nnz[jj] > ncrs) d_nnz[jj] = ncrs;
164: if (o_nnz[jj] > (M/cr_bs-ncrs)) o_nnz[jj] = M/cr_bs-ncrs;
165: }
167: MatCreate(comm, &tMat);
168: MatSetSizes(tMat, ncrs, ncrs,PETSC_DETERMINE, PETSC_DETERMINE);
169: MatSetType(tMat,MATAIJ);
170: MatSeqAIJSetPreallocation(tMat,0,d_nnz);
171: MatMPIAIJSetPreallocation(tMat,0,d_nnz,0,o_nnz);
172: PetscFree(d_nnz);
173: PetscFree(o_nnz);
175: for (ii = Istart_crs; ii < Iend_crs; ii++) {
176: PetscInt dest_row = ii/cr_bs;
177: MatGetRow(Cmat,ii,&ncols,&idx,&vals);
178: for (jj = 0; jj < ncols; jj++) {
179: PetscInt dest_col = idx[jj]/cr_bs;
180: PetscScalar v = 1.0;
181: MatSetValues(tMat,1,&dest_row,1,&dest_col,&v,ADD_VALUES);
182: }
183: MatRestoreRow(Cmat,ii,&ncols,&idx,&vals);
184: }
185: MatAssemblyBegin(tMat,MAT_FINAL_ASSEMBLY);
186: MatAssemblyEnd(tMat,MAT_FINAL_ASSEMBLY);
188: if (llev++ == -1) {
189: PetscViewer viewer; char fname[32];
190: PetscSNPrintf(fname,sizeof(fname),"part_mat_%D.mat",llev);
191: PetscViewerBinaryOpen(comm,fname,FILE_MODE_WRITE,&viewer);
192: MatView(tMat, viewer);
193: PetscViewerDestroy(&viewer);
194: }
196: MatConvert(tMat, MATMPIADJ, MAT_INITIAL_MATRIX, &adj);
198: MatDestroy(&tMat);
199: } /* create 'adj' */
201: { /* partition: get newproc_idx */
202: char prefix[256];
203: const char *pcpre;
204: const PetscInt *is_idx;
205: MatPartitioning mpart;
206: IS proc_is;
207: PetscInt targetPE;
209: MatPartitioningCreate(comm, &mpart);
210: MatPartitioningSetAdjacency(mpart, adj);
211: PCGetOptionsPrefix(pc, &pcpre);
212: PetscSNPrintf(prefix,sizeof(prefix),"%spc_gamg_",pcpre ? pcpre : "");
213: PetscObjectSetOptionsPrefix((PetscObject)mpart,prefix);
214: MatPartitioningSetFromOptions(mpart);
215: MatPartitioningSetNParts(mpart, new_size);
216: MatPartitioningApply(mpart, &proc_is);
217: MatPartitioningDestroy(&mpart);
219: /* collect IS info */
220: PetscMalloc1(ncrs_eq, &newproc_idx);
221: ISGetIndices(proc_is, &is_idx);
222: targetPE = 1; /* bring to "front" of machine */
223: /*targetPE = size/new_size;*/ /* spread partitioning across machine */
224: for (kk = jj = 0 ; kk < nloc_old ; kk++) {
225: for (ii = 0 ; ii < cr_bs ; ii++, jj++) {
226: newproc_idx[jj] = is_idx[kk] * targetPE; /* distribution */
227: }
228: }
229: ISRestoreIndices(proc_is, &is_idx);
230: ISDestroy(&proc_is);
231: }
232: MatDestroy(&adj);
234: ISCreateGeneral(comm, ncrs_eq, newproc_idx, PETSC_COPY_VALUES, &is_eq_newproc);
235: if (newproc_idx != 0) {
236: PetscFree(newproc_idx);
237: }
238: } else { /* simple aggreagtion of parts -- 'is_eq_newproc' */
240: PetscInt rfactor,targetPE;
241: /* find factor */
242: if (new_size == 1) rfactor = size; /* easy */
243: else {
244: PetscReal best_fact = 0.;
245: jj = -1;
246: for (kk = 1 ; kk <= size ; kk++) {
247: if (size%kk==0) { /* a candidate */
248: PetscReal nactpe = (PetscReal)size/(PetscReal)kk, fact = nactpe/(PetscReal)new_size;
249: if (fact > 1.0) fact = 1./fact; /* keep fact < 1 */
250: if (fact > best_fact) {
251: best_fact = fact; jj = kk;
252: }
253: }
254: }
255: if (jj != -1) rfactor = jj;
256: else rfactor = 1; /* does this happen .. a prime */
257: }
258: new_size = size/rfactor;
260: if (new_size==nactive) {
261: *a_Amat_crs = Cmat; /* output - no repartitioning or reduction, bail out because nested here */
262: PetscFree(counts);
263: if (pc_gamg->verbose>0) {
264: PetscPrintf(comm,"\t[%d]%s aggregate processors noop: new_size=%d, neq(loc)=%d\n",rank,__FUNCT__,new_size,ncrs_eq);
265: }
266: return(0);
267: }
269: if (pc_gamg->verbose) PetscPrintf(comm,"\t[%d]%s number of equations (loc) %d with simple aggregation\n",rank,__FUNCT__,ncrs_eq);
270: targetPE = rank/rfactor;
271: ISCreateStride(comm, ncrs_eq, targetPE, 0, &is_eq_newproc);
272: } /* end simple 'is_eq_newproc' */
274: /*
275: Create an index set from the is_eq_newproc index set to indicate the mapping TO
276: */
277: ISPartitioningToNumbering(is_eq_newproc, &is_eq_num);
278: is_eq_num_prim = is_eq_num;
279: /*
280: Determine how many equations/vertices are assigned to each processor
281: */
282: ISPartitioningCount(is_eq_newproc, size, counts);
283: ncrs_eq_new = counts[rank];
284: ISDestroy(&is_eq_newproc);
285: ncrs_new = ncrs_eq_new/cr_bs; /* eqs */
287: PetscFree(counts);
288: #if defined PETSC_GAMG_USE_LOG
289: PetscLogEventEnd(petsc_gamg_setup_events[SET12],0,0,0,0);
290: #endif
291: /* data movement scope -- this could be moved to subclasses so that we don't try to cram all auxilary data into some complex abstracted thing */
292: {
293: Vec src_crd, dest_crd;
294: const PetscInt *idx,ndata_rows=pc_gamg->data_cell_rows,ndata_cols=pc_gamg->data_cell_cols,node_data_sz=ndata_rows*ndata_cols;
295: VecScatter vecscat;
296: PetscScalar *array;
297: IS isscat;
299: /* move data (for primal equations only) */
300: /* Create a vector to contain the newly ordered element information */
301: VecCreate(comm, &dest_crd);
302: VecSetSizes(dest_crd, node_data_sz*ncrs_new, PETSC_DECIDE);
303: VecSetType(dest_crd,VECSTANDARD); /* this is needed! */
304: /*
305: There are 'ndata_rows*ndata_cols' data items per node, (one can think of the vectors of having
306: a block size of ...). Note, ISs are expanded into equation space by 'cr_bs'.
307: */
308: PetscMalloc1((ncrs*node_data_sz), &tidx);
309: ISGetIndices(is_eq_num_prim, &idx);
310: for (ii=0,jj=0; ii<ncrs; ii++) {
311: PetscInt id = idx[ii*cr_bs]/cr_bs; /* get node back */
312: for (kk=0; kk<node_data_sz; kk++, jj++) tidx[jj] = id*node_data_sz + kk;
313: }
314: ISRestoreIndices(is_eq_num_prim, &idx);
315: ISCreateGeneral(comm, node_data_sz*ncrs, tidx, PETSC_COPY_VALUES, &isscat);
316: PetscFree(tidx);
317: /*
318: Create a vector to contain the original vertex information for each element
319: */
320: VecCreateSeq(PETSC_COMM_SELF, node_data_sz*ncrs, &src_crd);
321: for (jj=0; jj<ndata_cols; jj++) {
322: const PetscInt stride0=ncrs*pc_gamg->data_cell_rows;
323: for (ii=0; ii<ncrs; ii++) {
324: for (kk=0; kk<ndata_rows; kk++) {
325: PetscInt ix = ii*ndata_rows + kk + jj*stride0, jx = ii*node_data_sz + kk*ndata_cols + jj;
326: PetscScalar tt = (PetscScalar)pc_gamg->data[ix];
327: VecSetValues(src_crd, 1, &jx, &tt, INSERT_VALUES);
328: }
329: }
330: }
331: VecAssemblyBegin(src_crd);
332: VecAssemblyEnd(src_crd);
333: /*
334: Scatter the element vertex information (still in the original vertex ordering)
335: to the correct processor
336: */
337: VecScatterCreate(src_crd, NULL, dest_crd, isscat, &vecscat);
338: ISDestroy(&isscat);
339: VecScatterBegin(vecscat,src_crd,dest_crd,INSERT_VALUES,SCATTER_FORWARD);
340: VecScatterEnd(vecscat,src_crd,dest_crd,INSERT_VALUES,SCATTER_FORWARD);
341: VecScatterDestroy(&vecscat);
342: VecDestroy(&src_crd);
343: /*
344: Put the element vertex data into a new allocation of the gdata->ele
345: */
346: PetscFree(pc_gamg->data);
347: PetscMalloc1(node_data_sz*ncrs_new, &pc_gamg->data);
349: pc_gamg->data_sz = node_data_sz*ncrs_new;
350: strideNew = ncrs_new*ndata_rows;
352: VecGetArray(dest_crd, &array);
353: for (jj=0; jj<ndata_cols; jj++) {
354: for (ii=0; ii<ncrs_new; ii++) {
355: for (kk=0; kk<ndata_rows; kk++) {
356: PetscInt ix = ii*ndata_rows + kk + jj*strideNew, jx = ii*node_data_sz + kk*ndata_cols + jj;
357: pc_gamg->data[ix] = PetscRealPart(array[jx]);
358: }
359: }
360: }
361: VecRestoreArray(dest_crd, &array);
362: VecDestroy(&dest_crd);
363: }
364: /* move A and P (columns) with new layout */
365: #if defined PETSC_GAMG_USE_LOG
366: PetscLogEventBegin(petsc_gamg_setup_events[SET13],0,0,0,0);
367: #endif
369: /*
370: Invert for MatGetSubMatrix
371: */
372: ISInvertPermutation(is_eq_num, ncrs_eq_new, &new_eq_indices);
373: ISSort(new_eq_indices); /* is this needed? */
374: ISSetBlockSize(new_eq_indices, cr_bs);
375: if (is_eq_num != is_eq_num_prim) {
376: ISDestroy(&is_eq_num_prim); /* could be same as 'is_eq_num' */
377: }
378: ISDestroy(&is_eq_num);
379: #if defined PETSC_GAMG_USE_LOG
380: PetscLogEventEnd(petsc_gamg_setup_events[SET13],0,0,0,0);
381: PetscLogEventBegin(petsc_gamg_setup_events[SET14],0,0,0,0);
382: #endif
383: /* 'a_Amat_crs' output */
384: {
385: Mat mat;
386: MatGetSubMatrix(Cmat, new_eq_indices, new_eq_indices, MAT_INITIAL_MATRIX, &mat);
387: *a_Amat_crs = mat;
389: if (!PETSC_TRUE) {
390: PetscInt cbs, rbs;
391: MatGetBlockSizes(Cmat, &rbs, &cbs);
392: PetscPrintf(MPI_COMM_SELF,"[%d]%s Old Mat rbs=%d cbs=%d\n",rank,__FUNCT__,rbs,cbs);
393: MatGetBlockSizes(mat, &rbs, &cbs);
394: PetscPrintf(MPI_COMM_SELF,"[%d]%s New Mat rbs=%d cbs=%d cr_bs=%d\n",rank,__FUNCT__,rbs,cbs,cr_bs);
395: }
396: }
397: MatDestroy(&Cmat);
399: #if defined PETSC_GAMG_USE_LOG
400: PetscLogEventEnd(petsc_gamg_setup_events[SET14],0,0,0,0);
401: #endif
402: /* prolongator */
403: {
404: IS findices;
405: PetscInt Istart,Iend;
406: Mat Pnew;
407: MatGetOwnershipRange(Pold, &Istart, &Iend);
408: #if defined PETSC_GAMG_USE_LOG
409: PetscLogEventBegin(petsc_gamg_setup_events[SET15],0,0,0,0);
410: #endif
411: ISCreateStride(comm,Iend-Istart,Istart,1,&findices);
412: ISSetBlockSize(findices,f_bs);
413: MatGetSubMatrix(Pold, findices, new_eq_indices, MAT_INITIAL_MATRIX, &Pnew);
414: ISDestroy(&findices);
416: if (!PETSC_TRUE) {
417: PetscInt cbs, rbs;
418: MatGetBlockSizes(Pold, &rbs, &cbs);
419: PetscPrintf(MPI_COMM_SELF,"[%d]%s Pold rbs=%d cbs=%d\n",rank,__FUNCT__,rbs,cbs);
420: MatGetBlockSizes(Pnew, &rbs, &cbs);
421: PetscPrintf(MPI_COMM_SELF,"[%d]%s Pnew rbs=%d cbs=%d\n",rank,__FUNCT__,rbs,cbs);
422: }
423: #if defined PETSC_GAMG_USE_LOG
424: PetscLogEventEnd(petsc_gamg_setup_events[SET15],0,0,0,0);
425: #endif
426: MatDestroy(a_P_inout);
428: /* output - repartitioned */
429: *a_P_inout = Pnew;
430: }
431: ISDestroy(&new_eq_indices);
433: *a_nactive_proc = new_size; /* output */
434: }
436: /* outout matrix data */
437: if (!PETSC_TRUE) {
438: PetscViewer viewer; char fname[32]; static int llev=0; Cmat = *a_Amat_crs;
439: if (llev==0) {
440: sprintf(fname,"Cmat_%d.m",llev++);
441: PetscViewerASCIIOpen(comm,fname,&viewer);
442: PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_MATLAB);
443: MatView(Amat_fine, viewer);
444: PetscViewerDestroy(&viewer);
445: }
446: sprintf(fname,"Cmat_%d.m",llev++);
447: PetscViewerASCIIOpen(comm,fname,&viewer);
448: PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_MATLAB);
449: MatView(Cmat, viewer);
450: PetscViewerDestroy(&viewer);
451: }
452: return(0);
453: }
455: /* -------------------------------------------------------------------------- */
456: /*
457: PCSetUp_GAMG - Prepares for the use of the GAMG preconditioner
458: by setting data structures and options.
460: Input Parameter:
461: . pc - the preconditioner context
463: Application Interface Routine: PCSetUp()
465: Notes:
466: The interface routine PCSetUp() is not usually called directly by
467: the user, but instead is called by PCApply() if necessary.
468: */
471: PetscErrorCode PCSetUp_GAMG(PC pc)
472: {
474: PC_MG *mg = (PC_MG*)pc->data;
475: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
476: Mat Pmat = pc->pmat;
477: PetscInt fine_level,level,level1,bs,M,qq,lidx,nASMBlocksArr[GAMG_MAXLEVELS];
478: MPI_Comm comm;
479: PetscMPIInt rank,size,nactivepe;
480: Mat Aarr[GAMG_MAXLEVELS],Parr[GAMG_MAXLEVELS];
481: PetscReal emaxs[GAMG_MAXLEVELS];
482: IS *ASMLocalIDsArr[GAMG_MAXLEVELS];
483: PetscLogDouble nnz0=0.,nnztot=0.;
484: MatInfo info;
485: PetscBool redo_mesh_setup = (PetscBool)(!pc_gamg->reuse_prol);
488: PetscObjectGetComm((PetscObject)pc,&comm);
489: MPI_Comm_rank(comm,&rank);
490: MPI_Comm_size(comm,&size);
492: if (pc_gamg->verbose>2) PetscPrintf(comm,"[%d]%s pc_gamg->setup_count=%d pc->setupcalled=%d\n",rank,__FUNCT__,pc_gamg->setup_count,pc->setupcalled);
494: if (pc_gamg->setup_count++ > 0) {
495: if (redo_mesh_setup) {
496: /* reset everything */
497: PCReset_MG(pc);
498: pc->setupcalled = 0;
499: } else {
500: PC_MG_Levels **mglevels = mg->levels;
501: /* just do Galerkin grids */
502: Mat B,dA,dB;
504: if (!pc->setupcalled) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"PCSetUp() has not been called yet");
505: if (pc_gamg->Nlevels > 1) {
506: /* currently only handle case where mat and pmat are the same on coarser levels */
507: KSPGetOperators(mglevels[pc_gamg->Nlevels-1]->smoothd,&dA,&dB,NULL);
508: /* (re)set to get dirty flag */
509: KSPSetOperators(mglevels[pc_gamg->Nlevels-1]->smoothd,dA,dB,SAME_NONZERO_PATTERN);
511: for (level=pc_gamg->Nlevels-2; level>=0; level--) {
512: /* the first time through the matrix structure has changed from repartitioning */
513: if (pc_gamg->setup_count==2) {
514: MatPtAP(dB,mglevels[level+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);
515: MatDestroy(&mglevels[level]->A);
517: mglevels[level]->A = B;
518: } else {
519: KSPGetOperators(mglevels[level]->smoothd,NULL,&B,NULL);
520: MatPtAP(dB,mglevels[level+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);
521: }
522: KSPSetOperators(mglevels[level]->smoothd,B,B,SAME_NONZERO_PATTERN);
523: dB = B;
524: }
525: }
527: PCSetUp_MG(pc);
529: /* PCSetUp_MG seems to insists on setting this to GMRES */
530: KSPSetType(mglevels[0]->smoothd, KSPPREONLY);
531: return(0);
532: }
533: }
535: if (!pc_gamg->data) {
536: if (pc_gamg->orig_data) {
537: MatGetBlockSize(Pmat, &bs);
538: MatGetLocalSize(Pmat, &qq, NULL);
540: pc_gamg->data_sz = (qq/bs)*pc_gamg->orig_data_cell_rows*pc_gamg->orig_data_cell_cols;
541: pc_gamg->data_cell_rows = pc_gamg->orig_data_cell_rows;
542: pc_gamg->data_cell_cols = pc_gamg->orig_data_cell_cols;
544: PetscMalloc1(pc_gamg->data_sz, &pc_gamg->data);
545: for (qq=0; qq<pc_gamg->data_sz; qq++) pc_gamg->data[qq] = pc_gamg->orig_data[qq];
546: } else {
547: if (!pc_gamg->ops->createdefaultdata) SETERRQ(comm,PETSC_ERR_PLIB,"'createdefaultdata' not set(?) need to support NULL data");
548: pc_gamg->ops->createdefaultdata(pc,Pmat);
549: }
550: }
552: /* cache original data for reuse */
553: if (!pc_gamg->orig_data && redo_mesh_setup) {
554: PetscMalloc1(pc_gamg->data_sz, &pc_gamg->orig_data);
555: for (qq=0; qq<pc_gamg->data_sz; qq++) pc_gamg->orig_data[qq] = pc_gamg->data[qq];
556: pc_gamg->orig_data_cell_rows = pc_gamg->data_cell_rows;
557: pc_gamg->orig_data_cell_cols = pc_gamg->data_cell_cols;
558: }
560: /* get basic dims */
561: MatGetBlockSize(Pmat, &bs);
563: MatGetSize(Pmat, &M, &qq);
564: if (pc_gamg->verbose) {
565: PetscInt NN = M;
566: if (pc_gamg->verbose==1) {
567: MatGetInfo(Pmat,MAT_LOCAL,&info);
568: MatGetLocalSize(Pmat, &NN, &qq);
569: } else {
570: MatGetInfo(Pmat,MAT_GLOBAL_SUM,&info);
571: }
572: nnz0 = info.nz_used;
573: nnztot = info.nz_used;
574: PetscPrintf(comm,"\t[%d]%s level %d N=%d, n data rows=%d, n data cols=%d, nnz/row (ave)=%d, np=%d\n",
575: rank,__FUNCT__,0,M,pc_gamg->data_cell_rows,pc_gamg->data_cell_cols,
576: (int)(nnz0/(PetscReal)NN),size);
577: }
579: /* Get A_i and R_i */
580: for (level=0, Aarr[0]=Pmat, nactivepe = size; /* hard wired stopping logic */
581: level < (pc_gamg->Nlevels-1) && (level==0 || M>pc_gamg->coarse_eq_limit); /* && (size==1 || nactivepe>1); */
582: level++) {
583: level1 = level + 1;
584: #if defined PETSC_GAMG_USE_LOG
585: PetscLogEventBegin(petsc_gamg_setup_events[SET1],0,0,0,0);
586: #if (defined GAMG_STAGES)
587: PetscLogStagePush(gamg_stages[level]);
588: #endif
589: #endif
590: { /* construct prolongator */
591: Mat Gmat;
592: PetscCoarsenData *agg_lists;
593: Mat Prol11;
595: pc_gamg->ops->graph(pc,Aarr[level], &Gmat);
596: pc_gamg->ops->coarsen(pc, &Gmat, &agg_lists);
597: pc_gamg->ops->prolongator(pc,Aarr[level],Gmat,agg_lists,&Prol11);
599: /* could have failed to create new level */
600: if (Prol11) {
601: /* get new block size of coarse matrices */
602: MatGetBlockSizes(Prol11, NULL, &bs);
604: if (pc_gamg->ops->optprol) {
605: /* smooth */
606: pc_gamg->ops->optprol(pc, Aarr[level], &Prol11);
607: }
609: Parr[level1] = Prol11;
610: } else Parr[level1] = NULL;
612: if (pc_gamg->use_aggs_in_gasm) {
613: PetscCDGetASMBlocks(agg_lists, bs, &nASMBlocksArr[level], &ASMLocalIDsArr[level]);
614: }
616: MatDestroy(&Gmat);
617: PetscCDDestroy(agg_lists);
618: } /* construct prolongator scope */
619: #if defined PETSC_GAMG_USE_LOG
620: PetscLogEventEnd(petsc_gamg_setup_events[SET1],0,0,0,0);
621: #endif
622: /* cache eigen estimate */
623: if (pc_gamg->emax_id != -1) {
624: PetscBool flag;
625: PetscObjectComposedDataGetReal((PetscObject)Aarr[level], pc_gamg->emax_id, emaxs[level], flag);
626: if (!flag) emaxs[level] = -1.;
627: } else emaxs[level] = -1.;
628: if (level==0) Aarr[0] = Pmat; /* use Pmat for finest level setup */
629: if (!Parr[level1]) {
630: if (pc_gamg->verbose) {
631: PetscPrintf(comm,"\t[%d]%s stop gridding, level %d\n",rank,__FUNCT__,level);
632: }
633: break;
634: }
635: #if defined PETSC_GAMG_USE_LOG
636: PetscLogEventBegin(petsc_gamg_setup_events[SET2],0,0,0,0);
637: #endif
639: createLevel(pc, Aarr[level], bs, (PetscBool)(level==pc_gamg->Nlevels-2),
640: &Parr[level1], &Aarr[level1], &nactivepe);
642: #if defined PETSC_GAMG_USE_LOG
643: PetscLogEventEnd(petsc_gamg_setup_events[SET2],0,0,0,0);
644: #endif
645: MatGetSize(Aarr[level1], &M, &qq);
647: if (pc_gamg->verbose > 0) {
648: PetscInt NN = M;
649: if (pc_gamg->verbose==1) {
650: MatGetInfo(Aarr[level1],MAT_LOCAL,&info);
651: MatGetLocalSize(Aarr[level1], &NN, &qq);
652: } else {
653: MatGetInfo(Aarr[level1], MAT_GLOBAL_SUM, &info);
654: }
656: nnztot += info.nz_used;
657: PetscPrintf(comm,"\t\t[%d]%s %d) N=%d, n data cols=%d, nnz/row (ave)=%d, %d active pes\n",
658: rank,__FUNCT__,(int)level1,M,pc_gamg->data_cell_cols,
659: (int)(info.nz_used/(PetscReal)NN), nactivepe);
660: }
662: /* stop if one node -- could pull back for singular problems */
663: if ( (pc_gamg->data_cell_cols && M/pc_gamg->data_cell_cols < 2) || (!pc_gamg->data_cell_cols && M < 2)) {
664: level++;
665: break;
666: }
667: #if (defined PETSC_GAMG_USE_LOG && defined GAMG_STAGES)
668: PetscLogStagePop();
669: #endif
670: } /* levels */
672: if (pc_gamg->data) {
673: PetscFree(pc_gamg->data);
674: pc_gamg->data = NULL;
675: }
677: if (pc_gamg->verbose) PetscPrintf(comm,"\t[%d]%s %d levels, grid complexity = %g\n",0,__FUNCT__,level+1,nnztot/nnz0);
678: pc_gamg->Nlevels = level + 1;
679: fine_level = level;
680: PCMGSetLevels(pc,pc_gamg->Nlevels,NULL);
682: /* simple setup */
683: if (!PETSC_TRUE) {
684: PC_MG_Levels **mglevels = mg->levels;
685: for (lidx=0,level=pc_gamg->Nlevels-1;
686: lidx<fine_level;
687: lidx++, level--) {
688: PCMGSetInterpolation(pc, lidx+1, Parr[level]);
689: KSPSetOperators(mglevels[lidx]->smoothd, Aarr[level], Aarr[level], SAME_NONZERO_PATTERN);
690: MatDestroy(&Parr[level]);
691: MatDestroy(&Aarr[level]);
692: }
693: KSPSetOperators(mglevels[fine_level]->smoothd, Aarr[0], Aarr[0], SAME_NONZERO_PATTERN);
695: PCSetUp_MG(pc);
696: } else if (pc_gamg->Nlevels > 1) { /* don't setup MG if one level */
697: /* set default smoothers & set operators */
698: for (lidx = 1, level = pc_gamg->Nlevels-2;
699: lidx <= fine_level;
700: lidx++, level--) {
701: KSP smoother;
702: PC subpc;
704: PCMGGetSmoother(pc, lidx, &smoother);
705: KSPGetPC(smoother, &subpc);
707: KSPSetNormType(smoother, KSP_NORM_NONE);
708: /* set ops */
709: KSPSetOperators(smoother, Aarr[level], Aarr[level], SAME_NONZERO_PATTERN);
710: PCMGSetInterpolation(pc, lidx, Parr[level+1]);
712: /* set defaults */
713: KSPSetType(smoother, KSPCHEBYSHEV);
715: /* override defaults and command line args (!) */
716: if (pc_gamg->use_aggs_in_gasm) {
717: PetscInt sz;
718: IS *is;
720: sz = nASMBlocksArr[level];
721: is = ASMLocalIDsArr[level];
722: PCSetType(subpc, PCGASM);
723: if (sz==0) {
724: IS is;
725: PetscInt my0,kk;
726: MatGetOwnershipRange(Aarr[level], &my0, &kk);
727: ISCreateGeneral(PETSC_COMM_SELF, 1, &my0, PETSC_COPY_VALUES, &is);
728: PCGASMSetSubdomains(subpc, 1, &is, NULL);
729: ISDestroy(&is);
730: } else {
731: PetscInt kk;
732: PCGASMSetSubdomains(subpc, sz, is, NULL);
733: for (kk=0; kk<sz; kk++) {
734: ISDestroy(&is[kk]);
735: }
736: PetscFree(is);
737: }
738: PCGASMSetOverlap(subpc, 0);
740: ASMLocalIDsArr[level] = NULL;
741: nASMBlocksArr[level] = 0;
742: PCGASMSetType(subpc, PC_GASM_BASIC);
743: } else {
744: PCSetType(subpc, PCSOR);
745: }
746: }
747: {
748: /* coarse grid */
749: KSP smoother,*k2; PC subpc,pc2; PetscInt ii,first;
750: Mat Lmat = Aarr[(level=pc_gamg->Nlevels-1)]; lidx = 0;
751: PCMGGetSmoother(pc, lidx, &smoother);
752: KSPSetOperators(smoother, Lmat, Lmat, SAME_NONZERO_PATTERN);
753: KSPSetNormType(smoother, KSP_NORM_NONE);
754: KSPGetPC(smoother, &subpc);
755: PCSetType(subpc, PCBJACOBI);
756: PCSetUp(subpc);
757: PCBJacobiGetSubKSP(subpc,&ii,&first,&k2);
758: if (ii != 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"ii %D is not one",ii);
759: KSPGetPC(k2[0],&pc2);
760: PCSetType(pc2, PCLU);
761: PCFactorSetShiftType(pc2,MAT_SHIFT_INBLOCKS);
762: KSPSetTolerances(k2[0],PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1);
763: /* This flag gets reset by PCBJacobiGetSubKSP(), but our BJacobi really does the same algorithm everywhere (and in
764: * fact, all but one process will have zero dofs), so we reset the flag to avoid having PCView_BJacobi attempt to
765: * view every subdomain as though they were different. */
766: ((PC_BJacobi*)subpc->data)->same_local_solves = PETSC_TRUE;
767: }
769: /* should be called in PCSetFromOptions_GAMG(), but cannot be called prior to PCMGSetLevels() */
770: PetscObjectOptionsBegin((PetscObject)pc);
771: PCSetFromOptions_MG(pc);
772: PetscOptionsEnd();
773: if (mg->galerkin != 2) SETERRQ(comm,PETSC_ERR_USER,"GAMG does Galerkin manually so the -pc_mg_galerkin option must not be used.");
775: /* create cheby smoothers */
776: for (lidx = 1, level = pc_gamg->Nlevels-2;
777: lidx <= fine_level;
778: lidx++, level--) {
779: KSP smoother;
780: PetscBool flag,flag2;
781: PC subpc;
783: PCMGGetSmoother(pc, lidx, &smoother);
784: KSPGetPC(smoother, &subpc);
786: /* do my own cheby */
787: PetscObjectTypeCompare((PetscObject)smoother, KSPCHEBYSHEV, &flag);
788: if (flag) {
789: PetscReal emax, emin;
790: PetscObjectTypeCompare((PetscObject)subpc, PCJACOBI, &flag);
791: PetscObjectTypeCompare((PetscObject)subpc, PCSOR, &flag2);
792: if ((flag||flag2) && emaxs[level] > 0.0) emax=emaxs[level]; /* eigen estimate only for diagnal PC but lets acccept SOR because it is close and safe (always lower) */
793: else { /* eigen estimate 'emax' -- this is done in cheby */
794: KSP eksp;
795: Mat Lmat = Aarr[level];
796: Vec bb, xx;
798: MatGetVecs(Lmat, &bb, 0);
799: MatGetVecs(Lmat, &xx, 0);
800: {
801: PetscRandom rctx;
802: PetscRandomCreate(comm,&rctx);
803: PetscRandomSetFromOptions(rctx);
804: VecSetRandom(bb,rctx);
805: PetscRandomDestroy(&rctx);
806: }
808: /* zeroing out BC rows -- needed for crazy matrices */
809: {
810: PetscInt Istart,Iend,ncols,jj,Ii;
811: PetscScalar zero = 0.0;
812: MatGetOwnershipRange(Lmat, &Istart, &Iend);
813: for (Ii = Istart, jj = 0; Ii < Iend; Ii++, jj++) {
814: MatGetRow(Lmat,Ii,&ncols,0,0);
815: if (ncols <= 1) {
816: VecSetValues(bb, 1, &Ii, &zero, INSERT_VALUES);
817: }
818: MatRestoreRow(Lmat,Ii,&ncols,0,0);
819: }
820: VecAssemblyBegin(bb);
821: VecAssemblyEnd(bb);
822: }
824: KSPCreate(comm, &eksp);
825: KSPSetTolerances(eksp, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT, 10);
826: KSPSetNormType(eksp, KSP_NORM_NONE);
827: KSPSetOptionsPrefix(eksp,((PetscObject)pc)->prefix);
828: KSPAppendOptionsPrefix(eksp, "gamg_est_");
829: KSPSetFromOptions(eksp);
831: KSPSetInitialGuessNonzero(eksp, PETSC_FALSE);
832: KSPSetOperators(eksp, Lmat, Lmat, SAME_NONZERO_PATTERN);
833: KSPSetComputeSingularValues(eksp,PETSC_TRUE);
835: /* set PC type to be same as smoother */
836: KSPSetPC(eksp, subpc);
838: /* solve - keep stuff out of logging */
839: PetscLogEventDeactivate(KSP_Solve);
840: PetscLogEventDeactivate(PC_Apply);
841: KSPSolve(eksp, bb, xx);
842: PetscLogEventActivate(KSP_Solve);
843: PetscLogEventActivate(PC_Apply);
845: KSPComputeExtremeSingularValues(eksp, &emax, &emin);
847: VecDestroy(&xx);
848: VecDestroy(&bb);
849: KSPDestroy(&eksp);
851: if (pc_gamg->verbose > 0) {
852: PetscInt N1, tt;
853: MatGetSize(Aarr[level], &N1, &tt);
854: PetscPrintf(comm,"\t\t\t%s PC setup max eigen=%e min=%e on level %d (N=%d)\n",__FUNCT__,emax,emin,lidx,N1);
855: }
856: }
857: {
858: PetscInt N1, N0;
859: MatGetSize(Aarr[level], &N1, NULL);
860: MatGetSize(Aarr[level+1], &N0, NULL);
861: /* heuristic - is this crap? */
862: /* emin = 1.*emax/((PetscReal)N1/(PetscReal)N0); */
863: emin = emax * pc_gamg->eigtarget[0];
864: emax *= pc_gamg->eigtarget[1];
865: }
866: KSPChebyshevSetEigenvalues(smoother, emax, emin);
867: } /* setup checby flag */
868: } /* non-coarse levels */
870: /* clean up */
871: for (level=1; level<pc_gamg->Nlevels; level++) {
872: MatDestroy(&Parr[level]);
873: MatDestroy(&Aarr[level]);
874: }
876: PCSetUp_MG(pc);
878: if (PETSC_TRUE) {
879: KSP smoother; /* PCSetUp_MG seems to insists on setting this to GMRES on coarse grid */
880: PCMGGetSmoother(pc, 0, &smoother);
881: KSPSetType(smoother, KSPPREONLY);
882: }
883: } else {
884: KSP smoother;
885: if (pc_gamg->verbose) PetscPrintf(comm,"\t[%d]%s one level solver used (system is seen as DD). Using default solver.\n",rank,__FUNCT__);
886: PCMGGetSmoother(pc, 0, &smoother);
887: KSPSetOperators(smoother, Aarr[0], Aarr[0], SAME_NONZERO_PATTERN);
888: KSPSetType(smoother, KSPPREONLY);
889: PCSetUp_MG(pc);
890: }
891: return(0);
892: }
894: /* ------------------------------------------------------------------------- */
895: /*
896: PCDestroy_GAMG - Destroys the private context for the GAMG preconditioner
897: that was created with PCCreate_GAMG().
899: Input Parameter:
900: . pc - the preconditioner context
902: Application Interface Routine: PCDestroy()
903: */
906: PetscErrorCode PCDestroy_GAMG(PC pc)
907: {
909: PC_MG *mg = (PC_MG*)pc->data;
910: PC_GAMG *pc_gamg= (PC_GAMG*)mg->innerctx;
913: PCReset_GAMG(pc);
914: if (pc_gamg->ops->destroy) {
915: (*pc_gamg->ops->destroy)(pc);
916: }
917: PetscFree(pc_gamg->ops);
918: PetscFree(pc_gamg->gamg_type_name);
919: PetscFree(pc_gamg);
920: PCDestroy_MG(pc);
921: return(0);
922: }
927: /*@
928: PCGAMGSetProcEqLim - Set number of equations to aim for on coarse grids via
929: processor reduction.
931: Not Collective on PC
933: Input Parameters:
934: . pc - the preconditioner context
937: Options Database Key:
938: . -pc_gamg_process_eq_limit
940: Level: intermediate
942: Concepts: Unstructured multrigrid preconditioner
944: .seealso: ()
945: @*/
946: PetscErrorCode PCGAMGSetProcEqLim(PC pc, PetscInt n)
947: {
952: PetscTryMethod(pc,"PCGAMGSetProcEqLim_C",(PC,PetscInt),(pc,n));
953: return(0);
954: }
958: static PetscErrorCode PCGAMGSetProcEqLim_GAMG(PC pc, PetscInt n)
959: {
960: PC_MG *mg = (PC_MG*)pc->data;
961: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
964: if (n>0) pc_gamg->min_eq_proc = n;
965: return(0);
966: }
970: /*@
971: PCGAMGSetCoarseEqLim - Set max number of equations on coarse grids.
973: Collective on PC
975: Input Parameters:
976: . pc - the preconditioner context
979: Options Database Key:
980: . -pc_gamg_coarse_eq_limit
982: Level: intermediate
984: Concepts: Unstructured multrigrid preconditioner
986: .seealso: ()
987: @*/
988: PetscErrorCode PCGAMGSetCoarseEqLim(PC pc, PetscInt n)
989: {
994: PetscTryMethod(pc,"PCGAMGSetCoarseEqLim_C",(PC,PetscInt),(pc,n));
995: return(0);
996: }
1000: static PetscErrorCode PCGAMGSetCoarseEqLim_GAMG(PC pc, PetscInt n)
1001: {
1002: PC_MG *mg = (PC_MG*)pc->data;
1003: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1006: if (n>0) pc_gamg->coarse_eq_limit = n;
1007: return(0);
1008: }
1012: /*@
1013: PCGAMGSetRepartitioning - Repartition the coarse grids
1015: Collective on PC
1017: Input Parameters:
1018: . pc - the preconditioner context
1021: Options Database Key:
1022: . -pc_gamg_repartition
1024: Level: intermediate
1026: Concepts: Unstructured multrigrid preconditioner
1028: .seealso: ()
1029: @*/
1030: PetscErrorCode PCGAMGSetRepartitioning(PC pc, PetscBool n)
1031: {
1036: PetscTryMethod(pc,"PCGAMGSetRepartitioning_C",(PC,PetscBool),(pc,n));
1037: return(0);
1038: }
1042: static PetscErrorCode PCGAMGSetRepartitioning_GAMG(PC pc, PetscBool n)
1043: {
1044: PC_MG *mg = (PC_MG*)pc->data;
1045: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1048: pc_gamg->repart = n;
1049: return(0);
1050: }
1054: /*@
1055: PCGAMGSetReuseProl - Reuse prlongation
1057: Collective on PC
1059: Input Parameters:
1060: . pc - the preconditioner context
1063: Options Database Key:
1064: . -pc_gamg_reuse_interpolation
1066: Level: intermediate
1068: Concepts: Unstructured multrigrid preconditioner
1070: .seealso: ()
1071: @*/
1072: PetscErrorCode PCGAMGSetReuseProl(PC pc, PetscBool n)
1073: {
1078: PetscTryMethod(pc,"PCGAMGSetReuseProl_C",(PC,PetscBool),(pc,n));
1079: return(0);
1080: }
1084: static PetscErrorCode PCGAMGSetReuseProl_GAMG(PC pc, PetscBool n)
1085: {
1086: PC_MG *mg = (PC_MG*)pc->data;
1087: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1090: pc_gamg->reuse_prol = n;
1091: return(0);
1092: }
1096: /*@
1097: PCGAMGSetUseASMAggs -
1099: Collective on PC
1101: Input Parameters:
1102: . pc - the preconditioner context
1105: Options Database Key:
1106: . -pc_gamg_use_agg_gasm
1108: Level: intermediate
1110: Concepts: Unstructured multrigrid preconditioner
1112: .seealso: ()
1113: @*/
1114: PetscErrorCode PCGAMGSetUseASMAggs(PC pc, PetscBool n)
1115: {
1120: PetscTryMethod(pc,"PCGAMGSetUseASMAggs_C",(PC,PetscBool),(pc,n));
1121: return(0);
1122: }
1126: static PetscErrorCode PCGAMGSetUseASMAggs_GAMG(PC pc, PetscBool n)
1127: {
1128: PC_MG *mg = (PC_MG*)pc->data;
1129: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1132: pc_gamg->use_aggs_in_gasm = n;
1133: return(0);
1134: }
1138: /*@
1139: PCGAMGSetNlevels -
1141: Not collective on PC
1143: Input Parameters:
1144: . pc - the preconditioner context
1147: Options Database Key:
1148: . -pc_mg_levels
1150: Level: intermediate
1152: Concepts: Unstructured multrigrid preconditioner
1154: .seealso: ()
1155: @*/
1156: PetscErrorCode PCGAMGSetNlevels(PC pc, PetscInt n)
1157: {
1162: PetscTryMethod(pc,"PCGAMGSetNlevels_C",(PC,PetscInt),(pc,n));
1163: return(0);
1164: }
1168: static PetscErrorCode PCGAMGSetNlevels_GAMG(PC pc, PetscInt n)
1169: {
1170: PC_MG *mg = (PC_MG*)pc->data;
1171: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1174: pc_gamg->Nlevels = n;
1175: return(0);
1176: }
1180: /*@
1181: PCGAMGSetThreshold - Relative threshold to use for dropping edges in aggregation graph
1183: Not collective on PC
1185: Input Parameters:
1186: . pc - the preconditioner context
1189: Options Database Key:
1190: . -pc_gamg_threshold
1192: Level: intermediate
1194: Concepts: Unstructured multrigrid preconditioner
1196: .seealso: ()
1197: @*/
1198: PetscErrorCode PCGAMGSetThreshold(PC pc, PetscReal n)
1199: {
1204: PetscTryMethod(pc,"PCGAMGSetThreshold_C",(PC,PetscReal),(pc,n));
1205: return(0);
1206: }
1210: static PetscErrorCode PCGAMGSetThreshold_GAMG(PC pc, PetscReal n)
1211: {
1212: PC_MG *mg = (PC_MG*)pc->data;
1213: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1216: pc_gamg->threshold = n;
1217: return(0);
1218: }
1222: /*@
1223: PCGAMGSetType - Set solution method - calls sub create method
1225: Collective on PC
1227: Input Parameters:
1228: . pc - the preconditioner context
1231: Options Database Key:
1232: . -pc_gamg_type
1234: Level: intermediate
1236: Concepts: Unstructured multrigrid preconditioner
1238: .seealso: ()
1239: @*/
1240: PetscErrorCode PCGAMGSetType(PC pc, PCGAMGType type)
1241: {
1246: PetscTryMethod(pc,"PCGAMGSetType_C",(PC,PCGAMGType),(pc,type));
1247: return(0);
1248: }
1252: static PetscErrorCode PCGAMGSetType_GAMG(PC pc, PCGAMGType type)
1253: {
1254: PetscErrorCode ierr,(*r)(PC);
1255: PC_MG *mg = (PC_MG*)pc->data;
1256: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1259: PetscFunctionListFind(GAMGList,type,&r);
1260: if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE,"Unknown GAMG type %s given",type);
1261: if (pc_gamg->ops->destroy) {
1262: /* there was something here - kill it */
1263: (*pc_gamg->ops->destroy)(pc);
1264: PetscMemzero(pc_gamg->ops,sizeof(struct _PCGAMGOps));
1265: /* cleaning up common data in pc_gamg - this should disapear someday */
1266: pc_gamg->data_cell_cols = 0;
1267: pc_gamg->data_cell_rows = 0;
1268: pc_gamg->orig_data_cell_cols = 0;
1269: pc_gamg->orig_data_cell_rows = 0;
1270: if (pc_gamg->data_sz) {
1271: PetscFree(pc_gamg->data);
1272: pc_gamg->data_sz = 0;
1273: }
1274: else if (pc_gamg->data) {
1275: PetscFree(pc_gamg->data); /* can this happen ? */
1276: }
1277: }
1278: PetscFree(pc_gamg->gamg_type_name);
1279: PetscStrallocpy(type,&pc_gamg->gamg_type_name);
1280: (*r)(pc);
1281: return(0);
1282: }
1286: PetscErrorCode PCSetFromOptions_GAMG(PC pc)
1287: {
1289: PC_MG *mg = (PC_MG*)pc->data;
1290: PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
1291: PetscBool flag;
1292: PetscInt two = 2;
1293: MPI_Comm comm;
1296: PetscObjectGetComm((PetscObject)pc,&comm);
1297: PetscOptionsHead("GAMG options");
1298: {
1299: /* -pc_gamg_type */
1300: {
1301: char tname[256];
1302: PetscOptionsFList("-pc_gamg_type","Type of AMG method","PCGAMGSetType",GAMGList, pc_gamg->gamg_type_name, tname, sizeof(tname), &flag);
1303: /* call PCCreateGAMG_XYZ */
1304: if (flag) {
1305: PCGAMGSetType(pc,tname);
1306: }
1307: }
1308: /* -pc_gamg_verbose */
1309: PetscOptionsInt("-pc_gamg_verbose","Verbose (debugging) output for PCGAMG",
1310: "none", pc_gamg->verbose,
1311: &pc_gamg->verbose, NULL);
1312: /* -pc_gamg_repartition */
1313: PetscOptionsBool("-pc_gamg_repartition",
1314: "Repartion coarse grids (false)",
1315: "PCGAMGRepartitioning",
1316: pc_gamg->repart,
1317: &pc_gamg->repart,
1318: &flag);
1319: /* -pc_gamg_reuse_interpolation */
1320: PetscOptionsBool("-pc_gamg_reuse_interpolation",
1321: "Reuse prolongation operator (true)",
1322: "PCGAMGReuseProl",
1323: pc_gamg->reuse_prol,
1324: &pc_gamg->reuse_prol,
1325: &flag);
1326: /* -pc_gamg_use_agg_gasm */
1327: PetscOptionsBool("-pc_gamg_use_agg_gasm",
1328: "Use aggregation agragates for GASM smoother (false)",
1329: "PCGAMGUseASMAggs",
1330: pc_gamg->use_aggs_in_gasm,
1331: &pc_gamg->use_aggs_in_gasm,
1332: &flag);
1333: /* -pc_gamg_process_eq_limit */
1334: PetscOptionsInt("-pc_gamg_process_eq_limit",
1335: "Limit (goal) on number of equations per process on coarse grids",
1336: "PCGAMGSetProcEqLim",
1337: pc_gamg->min_eq_proc,
1338: &pc_gamg->min_eq_proc,
1339: &flag);
1340: /* -pc_gamg_coarse_eq_limit */
1341: PetscOptionsInt("-pc_gamg_coarse_eq_limit",
1342: "Limit on number of equations for the coarse grid",
1343: "PCGAMGSetCoarseEqLim",
1344: pc_gamg->coarse_eq_limit,
1345: &pc_gamg->coarse_eq_limit,
1346: &flag);
1347: /* -pc_gamg_threshold */
1348: PetscOptionsReal("-pc_gamg_threshold",
1349: "Relative threshold to use for dropping edges in aggregation graph",
1350: "PCGAMGSetThreshold",
1351: pc_gamg->threshold,
1352: &pc_gamg->threshold,
1353: &flag);
1354: if (flag && pc_gamg->verbose) {
1355: PetscPrintf(comm,"\t[%d]%s threshold set %e\n",0,__FUNCT__,pc_gamg->threshold);
1356: }
1357: /* -pc_gamg_eigtarget */
1358: PetscOptionsRealArray("-pc_gamg_eigtarget","Target eigenvalue range as fraction of estimated maximum eigenvalue","PCGAMGSetEigTarget",pc_gamg->eigtarget,&two,NULL);
1359: PetscOptionsInt("-pc_mg_levels",
1360: "Set number of MG levels",
1361: "PCGAMGSetNlevels",
1362: pc_gamg->Nlevels,
1363: &pc_gamg->Nlevels,
1364: &flag);
1366: /* set options for subtype */
1367: if (pc_gamg->ops->setfromoptions) {(*pc_gamg->ops->setfromoptions)(pc);}
1368: }
1369: PetscOptionsTail();
1370: return(0);
1371: }
1373: /* -------------------------------------------------------------------------- */
1374: /*MC
1375: PCGAMG - Geometric algebraic multigrid (AMG) preconditioning framework.
1376: - This is the entry point to GAMG, registered in pcregis.c
1378: Options Database Keys:
1379: Multigrid options(inherited)
1380: + -pc_mg_cycles <1>: 1 for V cycle, 2 for W-cycle (PCMGSetCycleType)
1381: . -pc_mg_smoothup <1>: Number of post-smoothing steps (PCMGSetNumberSmoothUp)
1382: . -pc_mg_smoothdown <1>: Number of pre-smoothing steps (PCMGSetNumberSmoothDown)
1383: - -pc_mg_type <multiplicative>: (one of) additive multiplicative full kascade
1385: Level: intermediate
1387: Concepts: multigrid
1389: .seealso: PCCreate(), PCSetType(), PCType (for list of available types), PC, PCMGType,
1390: PCMGSetLevels(), PCMGGetLevels(), PCMGSetType(), PCMGSetCycleType(), PCMGSetNumberSmoothDown(),
1391: PCMGSetNumberSmoothUp(), PCMGGetCoarseSolve(), PCMGSetResidual(), PCMGSetInterpolation(),
1392: PCMGSetRestriction(), PCMGGetSmoother(), PCMGGetSmootherUp(), PCMGGetSmootherDown(),
1393: PCMGSetCyclesOnLevel(), PCMGSetRhs(), PCMGSetX(), PCMGSetR()
1394: M*/
1398: PETSC_EXTERN PetscErrorCode PCCreate_GAMG(PC pc)
1399: {
1401: PC_GAMG *pc_gamg;
1402: PC_MG *mg;
1403: #if defined PETSC_GAMG_USE_LOG
1404: static long count = 0;
1405: #endif
1408: /* PCGAMG is an inherited class of PCMG. Initialize pc as PCMG */
1409: PCSetType(pc, PCMG); /* calls PCCreate_MG() and MGCreate_Private() */
1410: PetscObjectChangeTypeName((PetscObject)pc, PCGAMG);
1412: /* create a supporting struct and attach it to pc */
1413: PetscNewLog(pc,&pc_gamg);
1414: mg = (PC_MG*)pc->data;
1415: mg->galerkin = 2; /* Use Galerkin, but it is computed externally */
1416: mg->innerctx = pc_gamg;
1418: PetscNewLog(pc,&pc_gamg->ops);
1420: pc_gamg->setup_count = 0;
1421: /* these should be in subctx but repartitioning needs simple arrays */
1422: pc_gamg->data_sz = 0;
1423: pc_gamg->data = 0;
1425: /* register AMG type */
1426: PCGAMGInitializePackage();
1428: /* overwrite the pointers of PCMG by the functions of base class PCGAMG */
1429: pc->ops->setfromoptions = PCSetFromOptions_GAMG;
1430: pc->ops->setup = PCSetUp_GAMG;
1431: pc->ops->reset = PCReset_GAMG;
1432: pc->ops->destroy = PCDestroy_GAMG;
1434: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetProcEqLim_C",PCGAMGSetProcEqLim_GAMG);
1435: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetCoarseEqLim_C",PCGAMGSetCoarseEqLim_GAMG);
1436: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetRepartitioning_C",PCGAMGSetRepartitioning_GAMG);
1437: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetReuseProl_C",PCGAMGSetReuseProl_GAMG);
1438: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetUseASMAggs_C",PCGAMGSetUseASMAggs_GAMG);
1439: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetThreshold_C",PCGAMGSetThreshold_GAMG);
1440: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetType_C",PCGAMGSetType_GAMG);
1441: PetscObjectComposeFunction((PetscObject)pc,"PCGAMGSetNlevels_C",PCGAMGSetNlevels_GAMG);
1442: pc_gamg->repart = PETSC_FALSE;
1443: pc_gamg->reuse_prol = PETSC_FALSE;
1444: pc_gamg->use_aggs_in_gasm = PETSC_FALSE;
1445: pc_gamg->min_eq_proc = 50;
1446: pc_gamg->coarse_eq_limit = 800;
1447: pc_gamg->threshold = 0.;
1448: pc_gamg->Nlevels = GAMG_MAXLEVELS;
1449: pc_gamg->verbose = 0;
1450: pc_gamg->emax_id = -1;
1451: pc_gamg->eigtarget[0] = 0.05;
1452: pc_gamg->eigtarget[1] = 1.05;
1454: /* private events */
1455: #if defined PETSC_GAMG_USE_LOG
1456: if (count++ == 0) {
1457: PetscLogEventRegister("GAMG: createProl", PC_CLASSID, &petsc_gamg_setup_events[SET1]);
1458: PetscLogEventRegister(" Graph", PC_CLASSID, &petsc_gamg_setup_events[GRAPH]);
1459: /* PetscLogEventRegister(" G.Mat", PC_CLASSID, &petsc_gamg_setup_events[GRAPH_MAT]); */
1460: /* PetscLogEventRegister(" G.Filter", PC_CLASSID, &petsc_gamg_setup_events[GRAPH_FILTER]); */
1461: /* PetscLogEventRegister(" G.Square", PC_CLASSID, &petsc_gamg_setup_events[GRAPH_SQR]); */
1462: PetscLogEventRegister(" MIS/Agg", PC_CLASSID, &petsc_gamg_setup_events[SET4]);
1463: PetscLogEventRegister(" geo: growSupp", PC_CLASSID, &petsc_gamg_setup_events[SET5]);
1464: PetscLogEventRegister(" geo: triangle", PC_CLASSID, &petsc_gamg_setup_events[SET6]);
1465: PetscLogEventRegister(" search&set", PC_CLASSID, &petsc_gamg_setup_events[FIND_V]);
1466: PetscLogEventRegister(" SA: col data", PC_CLASSID, &petsc_gamg_setup_events[SET7]);
1467: PetscLogEventRegister(" SA: frmProl0", PC_CLASSID, &petsc_gamg_setup_events[SET8]);
1468: PetscLogEventRegister(" SA: smooth", PC_CLASSID, &petsc_gamg_setup_events[SET9]);
1469: PetscLogEventRegister("GAMG: partLevel", PC_CLASSID, &petsc_gamg_setup_events[SET2]);
1470: PetscLogEventRegister(" repartition", PC_CLASSID, &petsc_gamg_setup_events[SET12]);
1471: PetscLogEventRegister(" Invert-Sort", PC_CLASSID, &petsc_gamg_setup_events[SET13]);
1472: PetscLogEventRegister(" Move A", PC_CLASSID, &petsc_gamg_setup_events[SET14]);
1473: PetscLogEventRegister(" Move P", PC_CLASSID, &petsc_gamg_setup_events[SET15]);
1475: /* PetscLogEventRegister(" PL move data", PC_CLASSID, &petsc_gamg_setup_events[SET13]); */
1476: /* PetscLogEventRegister("GAMG: fix", PC_CLASSID, &petsc_gamg_setup_events[SET10]); */
1477: /* PetscLogEventRegister("GAMG: set levels", PC_CLASSID, &petsc_gamg_setup_events[SET11]); */
1478: /* create timer stages */
1479: #if defined GAMG_STAGES
1480: {
1481: char str[32];
1482: PetscInt lidx;
1483: sprintf(str,"MG Level %d (finest)",0);
1484: PetscLogStageRegister(str, &gamg_stages[0]);
1485: for (lidx=1; lidx<9; lidx++) {
1486: sprintf(str,"MG Level %d",lidx);
1487: PetscLogStageRegister(str, &gamg_stages[lidx]);
1488: }
1489: }
1490: #endif
1491: }
1492: #endif
1493: /* PCSetUp_GAMG assumes that the type has been set, so set it to the default now */
1494: PCGAMGSetType(pc,PCGAMGAGG);
1495: return(0);
1496: }
1500: /*@C
1501: PCGAMGInitializePackage - This function initializes everything in the PCGAMG package. It is called
1502: from PetscDLLibraryRegister() when using dynamic libraries, and on the first call to PCCreate_GAMG()
1503: when using static libraries.
1505: Level: developer
1507: .keywords: PC, PCGAMG, initialize, package
1508: .seealso: PetscInitialize()
1509: @*/
1510: PetscErrorCode PCGAMGInitializePackage(void)
1511: {
1515: if (PCGAMGPackageInitialized) return(0);
1516: PCGAMGPackageInitialized = PETSC_TRUE;
1517: PetscFunctionListAdd(&GAMGList,PCGAMGGEO,PCCreateGAMG_GEO);
1518: PetscFunctionListAdd(&GAMGList,PCGAMGAGG,PCCreateGAMG_AGG);
1519: PetscFunctionListAdd(&GAMGList,PCGAMGCLASSICAL,PCCreateGAMG_Classical);
1520: PetscRegisterFinalize(PCGAMGFinalizePackage);
1522: /* general events */
1523: #if defined PETSC_USE_LOG
1524: PetscLogEventRegister("PCGAMGgraph_AGG", 0, &PC_GAMGGgraph_AGG);
1525: PetscLogEventRegister("PCGAMGgraph_GEO", PC_CLASSID, &PC_GAMGGgraph_GEO);
1526: PetscLogEventRegister("PCGAMGcoarse_AGG", PC_CLASSID, &PC_GAMGCoarsen_AGG);
1527: PetscLogEventRegister("PCGAMGcoarse_GEO", PC_CLASSID, &PC_GAMGCoarsen_GEO);
1528: PetscLogEventRegister("PCGAMGProl_AGG", PC_CLASSID, &PC_GAMGProlongator_AGG);
1529: PetscLogEventRegister("PCGAMGProl_GEO", PC_CLASSID, &PC_GAMGProlongator_GEO);
1530: PetscLogEventRegister("PCGAMGPOpt_AGG", PC_CLASSID, &PC_GAMGOptprol_AGG);
1531: #endif
1533: return(0);
1534: }
1538: /*@C
1539: PCGAMGFinalizePackage - This function destroys everything in the PCGAMG package. It is
1540: called from PetscFinalize().
1542: Level: developer
1544: .keywords: Petsc, destroy, package
1545: .seealso: PetscFinalize()
1546: @*/
1547: PetscErrorCode PCGAMGFinalizePackage(void)
1548: {
1552: PCGAMGPackageInitialized = PETSC_FALSE;
1553: PetscFunctionListDestroy(&GAMGList);
1554: return(0);
1555: }