Actual source code: mis.c
petsc-dev 2014-02-02
2: #include <petsc-private/matimpl.h> /*I "petscmat.h" I*/
3: #include <../src/mat/impls/aij/seq/aij.h>
4: #include <../src/mat/impls/aij/mpi/mpiaij.h>
7: /* typedef enum { NOT_DONE=-2, DELETED=-1, REMOVED=-3 } NState; */
8: /* use int instead of enum to facilitate passing them via Scatters */
9: typedef PetscInt NState;
10: static const NState NOT_DONE=-2;
11: static const NState DELETED =-1;
12: static const NState REMOVED =-3;
13: #define IS_SELECTED(s) (s!=DELETED && s!=NOT_DONE && s!=REMOVED)
15: /* -------------------------------------------------------------------------- */
16: /*
17: maxIndSetAgg - parallel maximal independent set (MIS) with data locality info. MatAIJ specific!!!
19: Input Parameter:
20: . perm - serial permutation of rows of local to process in MIS
21: . Gmat - glabal matrix of graph (data not defined)
22: . strict_aggs - flag for whether to keep strict (non overlapping) aggregates in 'llist';
23: . verbose -
24: Output Parameter:
25: . a_selected - IS of selected vertices, includes 'ghost' nodes at end with natural local indices
26: . a_locals_llist - array of list of nodes rooted at selected nodes
27: */
30: PetscErrorCode maxIndSetAgg(IS perm,Mat Gmat,PetscBool strict_aggs,PetscInt verbose,PetscCoarsenData **a_locals_llist)
31: {
32: PetscErrorCode ierr;
33: PetscBool isMPI;
34: Mat_SeqAIJ *matA, *matB = 0;
35: MPI_Comm comm;
36: Vec locState, ghostState;
37: PetscInt num_fine_ghosts,kk,n,ix,j,*idx,*ii,iter,Iend,my0,nremoved;
38: Mat_MPIAIJ *mpimat = 0;
39: PetscScalar *cpcol_gid,*cpcol_state;
40: PetscMPIInt mype,npe;
41: const PetscInt *perm_ix;
42: PetscInt nDone, nselected = 0;
43: const PetscInt nloc = Gmat->rmap->n;
44: PetscInt *lid_cprowID, *lid_gid;
45: PetscBool *lid_removed;
46: PetscScalar *lid_parent_gid = NULL; /* only used for strict aggs */
47: PetscScalar *lid_state;
48: PetscCoarsenData *agg_lists;
51: PetscObjectGetComm((PetscObject)Gmat,&comm);
52: MPI_Comm_rank(comm, &mype);
53: MPI_Comm_size(comm, &npe);
55: /* get submatrices */
56: PetscObjectTypeCompare((PetscObject)Gmat, MATMPIAIJ, &isMPI);
57: if (isMPI) {
58: mpimat = (Mat_MPIAIJ*)Gmat->data;
59: matA = (Mat_SeqAIJ*)mpimat->A->data;
60: matB = (Mat_SeqAIJ*)mpimat->B->data;
61: /* force compressed storage of B */
62: MatCheckCompressedRow(mpimat->B,matB->nonzerorowcnt,&matB->compressedrow,matB->i,Gmat->rmap->n,-1.0);
63: } else {
64: PetscBool isAIJ;
65: PetscObjectTypeCompare((PetscObject)Gmat, MATSEQAIJ, &isAIJ);
66: matA = (Mat_SeqAIJ*)Gmat->data;
67: }
68: /* get vector */
69: MatGetVecs(Gmat, &locState, 0);
71: MatGetOwnershipRange(Gmat,&my0,&Iend);
73: if (mpimat) {
74: PetscInt gid;
75: for (kk=0,gid=my0; kk<nloc; kk++,gid++) {
76: PetscScalar v = (PetscScalar)(gid);
77: VecSetValues(locState, 1, &gid, &v, INSERT_VALUES); /* set with GID */
78: }
79: VecAssemblyBegin(locState);
80: VecAssemblyEnd(locState);
81: VecScatterBegin(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
82: VecScatterEnd(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
83: VecGetArray(mpimat->lvec, &cpcol_gid); /* get proc ID in 'cpcol_gid' */
84: VecDuplicate(mpimat->lvec, &ghostState); /* need 2nd compressed col. of off proc data */
85: VecGetLocalSize(mpimat->lvec, &num_fine_ghosts);
86: VecSet(ghostState, (PetscScalar)((PetscReal)NOT_DONE)); /* set with UNKNOWN state */
87: } else num_fine_ghosts = 0;
89: PetscMalloc1(nloc, &lid_cprowID);
90: PetscMalloc1((nloc+1), &lid_gid); /* explicit array needed */
91: PetscMalloc1(nloc, &lid_removed); /* explicit array needed */
92: if (strict_aggs) {
93: PetscMalloc1((nloc+1), &lid_parent_gid);
94: }
95: PetscMalloc1((nloc+1), &lid_state);
97: /* has ghost nodes for !strict and uses local indexing (yuck) */
98: PetscCDCreate(strict_aggs ? nloc : num_fine_ghosts+nloc, &agg_lists);
99: if (a_locals_llist) *a_locals_llist = agg_lists;
101: /* need an inverse map - locals */
102: for (kk=0; kk<nloc; kk++) {
103: lid_cprowID[kk] = -1; lid_removed[kk] = PETSC_FALSE;
104: if (strict_aggs) {
105: lid_parent_gid[kk] = -1.0;
106: }
107: lid_gid[kk] = kk + my0;
108: lid_state[kk] = (PetscScalar)((PetscReal)NOT_DONE);
109: }
110: /* set index into cmpressed row 'lid_cprowID' */
111: if (matB) {
112: for (ix=0; ix<matB->compressedrow.nrows; ix++) {
113: PetscInt lid = matB->compressedrow.rindex[ix];
114: lid_cprowID[lid] = ix;
115: }
116: }
117: /* MIS */
118: iter = nremoved = nDone = 0;
119: ISGetIndices(perm, &perm_ix);
120: while (nDone < nloc || PETSC_TRUE) { /* asyncronous not implemented */
121: iter++;
122: if (mpimat) {
123: VecGetArray(ghostState, &cpcol_state);
124: }
125: /* check all vertices */
126: for (kk=0; kk<nloc; kk++) {
127: PetscInt lid = perm_ix[kk];
128: NState state = (NState)PetscRealPart(lid_state[lid]);
129: if (lid_removed[lid]) continue;
130: if (state == NOT_DONE) {
131: /* parallel test, delete if selected ghost */
132: PetscBool isOK = PETSC_TRUE;
133: if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
134: ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
135: idx = matB->j + ii[ix];
136: for (j=0; j<n; j++) {
137: PetscInt cpid = idx[j]; /* compressed row ID in B mat */
138: PetscInt gid = (PetscInt)PetscRealPart(cpcol_gid[cpid]);
139: NState statej = (NState)PetscRealPart(cpcol_state[cpid]);
140: if (statej == NOT_DONE && gid >= Iend) { /* should be (pe>mype), use gid as pe proxy */
141: isOK = PETSC_FALSE; /* can not delete */
142: break;
143: }
144: }
145: } /* parallel test */
146: if (isOK) { /* select or remove this vertex */
147: nDone++;
148: /* check for singleton */
149: ii = matA->i; n = ii[lid+1] - ii[lid];
150: if (n < 2) {
151: /* if I have any ghost adj then not a sing */
152: ix = lid_cprowID[lid];
153: if (ix==-1 || (matB->compressedrow.i[ix+1]-matB->compressedrow.i[ix])==0) {
154: nremoved++;
155: lid_removed[lid] = PETSC_TRUE;
156: /* should select this because it is technically in the MIS but lets not */
157: /* lid_state[lid] = (PetscScalar)(lid+my0); */
158: continue; /* one local adj (me) and no ghost - singleton */
159: }
160: }
161: /* SELECTED state encoded with global index */
162: lid_state[lid] = (PetscScalar)(lid+my0); /* needed???? */
163: nselected++;
164: if (strict_aggs) {
165: PetscCDAppendID(agg_lists, lid, lid+my0);
166: } else {
167: PetscCDAppendID(agg_lists, lid, lid);
168: }
169: /* delete local adj */
170: idx = matA->j + ii[lid];
171: for (j=0; j<n; j++) {
172: PetscInt lidj = idx[j];
173: NState statej = (NState)PetscRealPart(lid_state[lidj]);
174: if (statej == NOT_DONE) {
175: nDone++;
176: /* id_llist[lidj] = id_llist[lid]; id_llist[lid] = lidj; */ /* insert 'lidj' into head of llist */
177: if (strict_aggs) {
178: PetscCDAppendID(agg_lists, lid, lidj+my0);
179: } else {
180: PetscCDAppendID(agg_lists, lid, lidj);
181: }
182: lid_state[lidj] = (PetscScalar)(PetscReal)DELETED; /* delete this */
183: }
184: }
186: /* delete ghost adj of lid - deleted ghost done later for strict_aggs */
187: if (!strict_aggs) {
188: if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
189: ii = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
190: idx = matB->j + ii[ix];
191: for (j=0; j<n; j++) {
192: PetscInt cpid = idx[j]; /* compressed row ID in B mat */
193: NState statej = (NState)PetscRealPart(cpcol_state[cpid]);
194: if (statej == NOT_DONE) {
195: /* cpcol_state[cpid] = (PetscScalar)DELETED; this should happen later ... */
196: /* id_llist[lidj] = id_llist[lid]; id_llist[lid] = lidj; */ /* insert 'lidj' into head of llist */
197: PetscCDAppendID(agg_lists, lid, nloc+cpid);
198: }
199: }
200: }
201: }
202: } /* selected */
203: } /* not done vertex */
204: } /* vertex loop */
206: /* update ghost states and count todos */
207: if (mpimat) {
208: VecRestoreArray(ghostState, &cpcol_state);
209: /* put lid state in 'locState' */
210: VecSetValues(locState, nloc, lid_gid, lid_state, INSERT_VALUES);
211: VecAssemblyBegin(locState);
212: VecAssemblyEnd(locState);
213: /* scatter states, check for done */
214: VecScatterBegin(mpimat->Mvctx,locState,ghostState,INSERT_VALUES,SCATTER_FORWARD);
215: VecScatterEnd(mpimat->Mvctx,locState,ghostState,INSERT_VALUES,SCATTER_FORWARD);
216: /* delete locals from selected ghosts */
217: VecGetArray(ghostState, &cpcol_state);
218: ii = matB->compressedrow.i;
219: for (ix=0; ix<matB->compressedrow.nrows; ix++) {
220: PetscInt lid = matB->compressedrow.rindex[ix]; /* local boundary node */
221: NState state = (NState)PetscRealPart(lid_state[lid]);
222: if (state == NOT_DONE) {
223: /* look at ghosts */
224: n = ii[ix+1] - ii[ix];
225: idx = matB->j + ii[ix];
226: for (j=0; j<n; j++) {
227: PetscInt cpid = idx[j]; /* compressed row ID in B mat */
228: NState statej = (NState)PetscRealPart(cpcol_state[cpid]);
229: if (IS_SELECTED(statej)) { /* lid is now deleted, do it */
230: nDone++;
231: lid_state[lid] = (PetscScalar)(PetscReal)DELETED; /* delete this */
232: if (!strict_aggs) {
233: PetscInt lidj = nloc + cpid;
234: /* id_llist[lid] = id_llist[lidj]; id_llist[lidj] = lid; */ /* insert 'lid' into head of ghost llist */
235: PetscCDAppendID(agg_lists, lidj, lid);
236: } else {
237: PetscInt sgid = (PetscInt)PetscRealPart(cpcol_gid[cpid]);
238: lid_parent_gid[lid] = (PetscScalar)sgid; /* keep track of proc that I belong to */
239: }
240: break;
241: }
242: }
243: }
244: }
245: VecRestoreArray(ghostState, &cpcol_state);
247: /* all done? */
248: {
249: PetscInt t1, t2;
250: t1 = nloc - nDone;
251: MPI_Allreduce(&t1, &t2, 1, MPIU_INT, MPI_SUM, comm); /* synchronous version */
252: if (t2 == 0) break;
253: }
254: } else break; /* all done */
255: } /* outer parallel MIS loop */
256: ISRestoreIndices(perm,&perm_ix);
258: if (verbose) {
259: if (verbose == 1) {
260: PetscPrintf(comm,"\t[%d]%s removed %d of %d vertices. %d selected.\n",mype,__FUNCT__,nremoved,nloc,nselected);
261: } else {
262: MPI_Allreduce(&nremoved, &n, 1, MPIU_INT, MPI_SUM, comm);
263: MatGetSize(Gmat, &kk, &j);
264: MPI_Allreduce(&nselected, &j, 1, MPIU_INT, MPI_SUM, comm);
265: PetscPrintf(comm,"\t[%d]%s removed %d of %d vertices. (%d local) %d selected.\n",mype,__FUNCT__,n,kk,nremoved,j);
266: }
267: }
269: /* tell adj who my lid_parent_gid vertices belong to - fill in agg_lists selected ghost lists */
270: if (strict_aggs && matB) {
271: PetscScalar *cpcol_sel_gid;
272: PetscInt cpid,*icpcol_gid;
274: /* need to copy this to free buffer -- should do this globaly */
275: PetscMalloc1(num_fine_ghosts, &icpcol_gid);
276: for (cpid=0; cpid<num_fine_ghosts; cpid++) icpcol_gid[cpid] = (PetscInt)PetscRealPart(cpcol_gid[cpid]);
278: /* get proc of deleted ghost */
279: VecSetValues(locState, nloc, lid_gid, lid_parent_gid, INSERT_VALUES);
280: VecAssemblyBegin(locState);
281: VecAssemblyEnd(locState);
282: VecScatterBegin(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
283: VecScatterEnd(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
284: VecGetArray(mpimat->lvec, &cpcol_sel_gid); /* has pe that owns ghost */
285: for (cpid=0; cpid<num_fine_ghosts; cpid++) {
286: PetscInt sgid = (PetscInt)PetscRealPart(cpcol_sel_gid[cpid]);
287: PetscInt gid = icpcol_gid[cpid];
288: if (sgid >= my0 && sgid < Iend) { /* I own this deleted */
289: PetscInt slid = sgid - my0;
290: /* id_llist[lidj] = id_llist[lid]; id_llist[lid] = lidj; */ /* insert 'lidj' into head of llist */
291: PetscCDAppendID(agg_lists, slid, gid);
292: }
293: }
294: VecRestoreArray(mpimat->lvec, &cpcol_sel_gid);
295: PetscFree(icpcol_gid);
296: } else if (matB) {
297: VecRestoreArray(mpimat->lvec, &cpcol_gid);
298: }
300: /* cache IS of removed nodes, use 'lid_gid' */
301: /* for (kk=n=0,ix=my0;kk<nloc;kk++,ix++) { */
302: /* if (lid_removed[kk]) lid_gid[n++] = ix; */
303: /* } */
304: /* PetscCDSetRemovedIS(agg_lists, comm, n, lid_gid); */
306: PetscFree(lid_cprowID);
307: PetscFree(lid_gid);
308: PetscFree(lid_removed);
309: if (strict_aggs) {
310: PetscFree(lid_parent_gid);
311: }
312: PetscFree(lid_state);
314: if (mpimat) {
315: VecDestroy(&ghostState);
316: }
317: VecDestroy(&locState);
318: return(0);
319: }
321: typedef struct {
322: int dummy;
323: } MatCoarsen_MIS;
324: /*
325: MIS coarsen, simple greedy.
326: */
329: static PetscErrorCode MatCoarsenApply_MIS(MatCoarsen coarse)
330: {
331: /* MatCoarsen_MIS *MIS = (MatCoarsen_MIS*)coarse->; */
333: Mat mat = coarse->graph;
337: if (!coarse->perm) {
338: IS perm;
339: PetscInt n,m;
340: MPI_Comm comm;
341: PetscObjectGetComm((PetscObject)mat,&comm);
342: MatGetLocalSize(mat, &m, &n);
343: ISCreateStride(comm, m, 0, 1, &perm);
344: maxIndSetAgg(perm, mat, coarse->strict_aggs, coarse->verbose, &coarse->agg_lists);
345: ISDestroy(&perm);
346: } else {
347: maxIndSetAgg(coarse->perm, mat, coarse->strict_aggs, coarse->verbose, &coarse->agg_lists);
348: }
349: return(0);
350: }
354: PetscErrorCode MatCoarsenView_MIS(MatCoarsen coarse,PetscViewer viewer)
355: {
356: /* MatCoarsen_MIS *MIS = (MatCoarsen_MIS*)coarse->; */
358: PetscMPIInt rank;
359: PetscBool iascii;
363: MPI_Comm_rank(PetscObjectComm((PetscObject)coarse),&rank);
364: PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
365: if (iascii) {
366: PetscViewerASCIISynchronizedPrintf(viewer," [%d] MIS aggregator\n",rank);
367: PetscViewerFlush(viewer);
368: PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
369: }
370: return(0);
371: }
375: PetscErrorCode MatCoarsenDestroy_MIS(MatCoarsen coarse)
376: {
377: MatCoarsen_MIS *MIS = (MatCoarsen_MIS*)coarse->subctx;
382: PetscFree(MIS);
383: return(0);
384: }
386: /*MC
387: MATCOARSENMIS - Creates a coarsen context via the external package MIS.
389: Collective on MPI_Comm
391: Input Parameter:
392: . coarse - the coarsen context
394: Options Database Keys:
395: + -mat_coarsen_MIS_xxx -
397: Level: beginner
399: .keywords: Coarsen, create, context
401: .seealso: MatCoarsenSetType(), MatCoarsenType
403: M*/
407: PETSC_EXTERN PetscErrorCode MatCoarsenCreate_MIS(MatCoarsen coarse)
408: {
410: MatCoarsen_MIS *MIS;
413: PetscNewLog(coarse,&MIS);
414: coarse->subctx = (void*)MIS;
416: coarse->ops->apply = MatCoarsenApply_MIS;
417: coarse->ops->view = MatCoarsenView_MIS;
418: coarse->ops->destroy = MatCoarsenDestroy_MIS;
419: /* coarse->ops->setfromoptions = MatCoarsenSetFromOptions_MIS; */
420: return(0);
421: }