Actual source code: mis.c

petsc-dev 2014-02-02
Report Typos and Errors
  2: #include <petsc-private/matimpl.h>    /*I "petscmat.h" I*/
  3: #include <../src/mat/impls/aij/seq/aij.h>
  4: #include <../src/mat/impls/aij/mpi/mpiaij.h>


  7: /* typedef enum { NOT_DONE=-2, DELETED=-1, REMOVED=-3 } NState; */
  8: /* use int instead of enum to facilitate passing them via Scatters */
  9: typedef PetscInt NState;
 10: static const NState NOT_DONE=-2;
 11: static const NState DELETED =-1;
 12: static const NState REMOVED =-3;
 13: #define IS_SELECTED(s) (s!=DELETED && s!=NOT_DONE && s!=REMOVED)

 15: /* -------------------------------------------------------------------------- */
 16: /*
 17:    maxIndSetAgg - parallel maximal independent set (MIS) with data locality info. MatAIJ specific!!!

 19:    Input Parameter:
 20:    . perm - serial permutation of rows of local to process in MIS
 21:    . Gmat - glabal matrix of graph (data not defined)
 22:    . strict_aggs - flag for whether to keep strict (non overlapping) aggregates in 'llist';
 23:    . verbose -
 24:    Output Parameter:
 25:    . a_selected - IS of selected vertices, includes 'ghost' nodes at end with natural local indices
 26:    . a_locals_llist - array of list of nodes rooted at selected nodes
 27: */
 30: PetscErrorCode maxIndSetAgg(IS perm,Mat Gmat,PetscBool strict_aggs,PetscInt verbose,PetscCoarsenData **a_locals_llist)
 31: {
 32:   PetscErrorCode   ierr;
 33:   PetscBool        isMPI;
 34:   Mat_SeqAIJ       *matA, *matB = 0;
 35:   MPI_Comm         comm;
 36:   Vec              locState, ghostState;
 37:   PetscInt         num_fine_ghosts,kk,n,ix,j,*idx,*ii,iter,Iend,my0,nremoved;
 38:   Mat_MPIAIJ       *mpimat = 0;
 39:   PetscScalar      *cpcol_gid,*cpcol_state;
 40:   PetscMPIInt      mype,npe;
 41:   const PetscInt   *perm_ix;
 42:   PetscInt         nDone, nselected = 0;
 43:   const PetscInt   nloc = Gmat->rmap->n;
 44:   PetscInt         *lid_cprowID, *lid_gid;
 45:   PetscBool        *lid_removed;
 46:   PetscScalar      *lid_parent_gid = NULL; /* only used for strict aggs */
 47:   PetscScalar      *lid_state;
 48:   PetscCoarsenData *agg_lists;

 51:   PetscObjectGetComm((PetscObject)Gmat,&comm);
 52:   MPI_Comm_rank(comm, &mype);
 53:   MPI_Comm_size(comm, &npe);

 55:   /* get submatrices */
 56:   PetscObjectTypeCompare((PetscObject)Gmat, MATMPIAIJ, &isMPI);
 57:   if (isMPI) {
 58:     mpimat = (Mat_MPIAIJ*)Gmat->data;
 59:     matA   = (Mat_SeqAIJ*)mpimat->A->data;
 60:     matB   = (Mat_SeqAIJ*)mpimat->B->data;
 61:     /* force compressed storage of B */
 62:     MatCheckCompressedRow(mpimat->B,matB->nonzerorowcnt,&matB->compressedrow,matB->i,Gmat->rmap->n,-1.0);
 63:   } else {
 64:     PetscBool isAIJ;
 65:     PetscObjectTypeCompare((PetscObject)Gmat, MATSEQAIJ, &isAIJ);
 66:     matA = (Mat_SeqAIJ*)Gmat->data;
 67:   }
 68:   /* get vector */
 69:   MatGetVecs(Gmat, &locState, 0);

 71:   MatGetOwnershipRange(Gmat,&my0,&Iend);

 73:   if (mpimat) {
 74:     PetscInt gid;
 75:     for (kk=0,gid=my0; kk<nloc; kk++,gid++) {
 76:       PetscScalar v = (PetscScalar)(gid);
 77:       VecSetValues(locState, 1, &gid, &v, INSERT_VALUES); /* set with GID */
 78:     }
 79:     VecAssemblyBegin(locState);
 80:     VecAssemblyEnd(locState);
 81:     VecScatterBegin(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
 82:       VecScatterEnd(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
 83:     VecGetArray(mpimat->lvec, &cpcol_gid); /* get proc ID in 'cpcol_gid' */
 84:     VecDuplicate(mpimat->lvec, &ghostState); /* need 2nd compressed col. of off proc data */
 85:     VecGetLocalSize(mpimat->lvec, &num_fine_ghosts);
 86:     VecSet(ghostState, (PetscScalar)((PetscReal)NOT_DONE)); /* set with UNKNOWN state */
 87:   } else num_fine_ghosts = 0;

 89:   PetscMalloc1(nloc, &lid_cprowID);
 90:   PetscMalloc1((nloc+1), &lid_gid); /* explicit array needed */
 91:   PetscMalloc1(nloc, &lid_removed); /* explicit array needed */
 92:   if (strict_aggs) {
 93:     PetscMalloc1((nloc+1), &lid_parent_gid);
 94:   }
 95:   PetscMalloc1((nloc+1), &lid_state);

 97:   /* has ghost nodes for !strict and uses local indexing (yuck) */
 98:   PetscCDCreate(strict_aggs ? nloc : num_fine_ghosts+nloc, &agg_lists);
 99:   if (a_locals_llist) *a_locals_llist = agg_lists;

101:   /* need an inverse map - locals */
102:   for (kk=0; kk<nloc; kk++) {
103:     lid_cprowID[kk] = -1; lid_removed[kk] = PETSC_FALSE;
104:     if (strict_aggs) {
105:       lid_parent_gid[kk] = -1.0;
106:     }
107:     lid_gid[kk]   = kk + my0;
108:     lid_state[kk] = (PetscScalar)((PetscReal)NOT_DONE);
109:   }
110:   /* set index into cmpressed row 'lid_cprowID' */
111:   if (matB) {
112:     for (ix=0; ix<matB->compressedrow.nrows; ix++) {
113:       PetscInt lid = matB->compressedrow.rindex[ix];
114:       lid_cprowID[lid] = ix;
115:     }
116:   }
117:   /* MIS */
118:   iter = nremoved = nDone = 0;
119:   ISGetIndices(perm, &perm_ix);
120:   while (nDone < nloc || PETSC_TRUE) { /* asyncronous not implemented */
121:     iter++;
122:     if (mpimat) {
123:       VecGetArray(ghostState, &cpcol_state);
124:     }
125:     /* check all vertices */
126:     for (kk=0; kk<nloc; kk++) {
127:       PetscInt lid   = perm_ix[kk];
128:       NState   state = (NState)PetscRealPart(lid_state[lid]);
129:       if (lid_removed[lid]) continue;
130:       if (state == NOT_DONE) {
131:         /* parallel test, delete if selected ghost */
132:         PetscBool isOK = PETSC_TRUE;
133:         if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
134:           ii  = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
135:           idx = matB->j + ii[ix];
136:           for (j=0; j<n; j++) {
137:             PetscInt cpid   = idx[j]; /* compressed row ID in B mat */
138:             PetscInt gid    = (PetscInt)PetscRealPart(cpcol_gid[cpid]);
139:             NState   statej = (NState)PetscRealPart(cpcol_state[cpid]);
140:             if (statej == NOT_DONE && gid >= Iend) { /* should be (pe>mype), use gid as pe proxy */
141:               isOK = PETSC_FALSE; /* can not delete */
142:               break;
143:             }
144:           }
145:         } /* parallel test */
146:         if (isOK) { /* select or remove this vertex */
147:           nDone++;
148:           /* check for singleton */
149:           ii = matA->i; n = ii[lid+1] - ii[lid];
150:           if (n < 2) {
151:             /* if I have any ghost adj then not a sing */
152:             ix = lid_cprowID[lid];
153:             if (ix==-1 || (matB->compressedrow.i[ix+1]-matB->compressedrow.i[ix])==0) {
154:               nremoved++;
155:               lid_removed[lid] = PETSC_TRUE;
156:               /* should select this because it is technically in the MIS but lets not */
157:               /* lid_state[lid] = (PetscScalar)(lid+my0); */
158:               continue; /* one local adj (me) and no ghost - singleton */
159:             }
160:           }
161:           /* SELECTED state encoded with global index */
162:           lid_state[lid] = (PetscScalar)(lid+my0); /* needed???? */
163:           nselected++;
164:           if (strict_aggs) {
165:             PetscCDAppendID(agg_lists, lid, lid+my0);
166:           } else {
167:             PetscCDAppendID(agg_lists, lid, lid);
168:           }
169:           /* delete local adj */
170:           idx = matA->j + ii[lid];
171:           for (j=0; j<n; j++) {
172:             PetscInt lidj   = idx[j];
173:             NState   statej = (NState)PetscRealPart(lid_state[lidj]);
174:             if (statej == NOT_DONE) {
175:               nDone++;
176:               /* id_llist[lidj] = id_llist[lid]; id_llist[lid] = lidj; */ /* insert 'lidj' into head of llist */
177:               if (strict_aggs) {
178:                 PetscCDAppendID(agg_lists, lid, lidj+my0);
179:               } else {
180:                 PetscCDAppendID(agg_lists, lid, lidj);
181:               }
182:               lid_state[lidj] = (PetscScalar)(PetscReal)DELETED;  /* delete this */
183:             }
184:           }

186:           /* delete ghost adj of lid - deleted ghost done later for strict_aggs */
187:           if (!strict_aggs) {
188:             if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
189:               ii  = matB->compressedrow.i; n = ii[ix+1] - ii[ix];
190:               idx = matB->j + ii[ix];
191:               for (j=0; j<n; j++) {
192:                 PetscInt cpid   = idx[j]; /* compressed row ID in B mat */
193:                 NState   statej = (NState)PetscRealPart(cpcol_state[cpid]);
194:                 if (statej == NOT_DONE) {
195:                   /* cpcol_state[cpid] = (PetscScalar)DELETED; this should happen later ... */
196:                   /* id_llist[lidj] = id_llist[lid]; id_llist[lid] = lidj; */ /* insert 'lidj' into head of llist */
197:                   PetscCDAppendID(agg_lists, lid, nloc+cpid);
198:                 }
199:               }
200:             }
201:           }
202:         } /* selected */
203:       } /* not done vertex */
204:     } /* vertex loop */

206:     /* update ghost states and count todos */
207:     if (mpimat) {
208:       VecRestoreArray(ghostState, &cpcol_state);
209:       /* put lid state in 'locState' */
210:       VecSetValues(locState, nloc, lid_gid, lid_state, INSERT_VALUES);
211:       VecAssemblyBegin(locState);
212:       VecAssemblyEnd(locState);
213:       /* scatter states, check for done */
214:       VecScatterBegin(mpimat->Mvctx,locState,ghostState,INSERT_VALUES,SCATTER_FORWARD);
215:         VecScatterEnd(mpimat->Mvctx,locState,ghostState,INSERT_VALUES,SCATTER_FORWARD);
216:       /* delete locals from selected ghosts */
217:       VecGetArray(ghostState, &cpcol_state);
218:       ii   = matB->compressedrow.i;
219:       for (ix=0; ix<matB->compressedrow.nrows; ix++) {
220:         PetscInt lid   = matB->compressedrow.rindex[ix]; /* local boundary node */
221:         NState   state = (NState)PetscRealPart(lid_state[lid]);
222:         if (state == NOT_DONE) {
223:           /* look at ghosts */
224:           n   = ii[ix+1] - ii[ix];
225:           idx = matB->j + ii[ix];
226:           for (j=0; j<n; j++) {
227:             PetscInt cpid   = idx[j]; /* compressed row ID in B mat */
228:             NState   statej = (NState)PetscRealPart(cpcol_state[cpid]);
229:             if (IS_SELECTED(statej)) { /* lid is now deleted, do it */
230:               nDone++;
231:               lid_state[lid] = (PetscScalar)(PetscReal)DELETED; /* delete this */
232:               if (!strict_aggs) {
233:                 PetscInt lidj = nloc + cpid;
234:                 /* id_llist[lid] = id_llist[lidj]; id_llist[lidj] = lid; */ /* insert 'lid' into head of ghost llist */
235:                 PetscCDAppendID(agg_lists, lidj, lid);
236:               } else {
237:                 PetscInt sgid = (PetscInt)PetscRealPart(cpcol_gid[cpid]);
238:                 lid_parent_gid[lid] = (PetscScalar)sgid; /* keep track of proc that I belong to */
239:               }
240:               break;
241:             }
242:           }
243:         }
244:       }
245:       VecRestoreArray(ghostState, &cpcol_state);

247:       /* all done? */
248:       {
249:         PetscInt t1, t2;
250:         t1   = nloc - nDone;
251:         MPI_Allreduce(&t1, &t2, 1, MPIU_INT, MPI_SUM, comm); /* synchronous version */
252:         if (t2 == 0) break;
253:       }
254:     } else break; /* all done */
255:   } /* outer parallel MIS loop */
256:   ISRestoreIndices(perm,&perm_ix);

258:   if (verbose) {
259:     if (verbose == 1) {
260:       PetscPrintf(comm,"\t[%d]%s removed %d of %d vertices.  %d selected.\n",mype,__FUNCT__,nremoved,nloc,nselected);
261:     } else {
262:       MPI_Allreduce(&nremoved, &n, 1, MPIU_INT, MPI_SUM, comm);
263:       MatGetSize(Gmat, &kk, &j);
264:       MPI_Allreduce(&nselected, &j, 1, MPIU_INT, MPI_SUM, comm);
265:       PetscPrintf(comm,"\t[%d]%s removed %d of %d vertices. (%d local)  %d selected.\n",mype,__FUNCT__,n,kk,nremoved,j);
266:     }
267:   }

269:   /* tell adj who my lid_parent_gid vertices belong to - fill in agg_lists selected ghost lists */
270:   if (strict_aggs && matB) {
271:     PetscScalar *cpcol_sel_gid;
272:     PetscInt    cpid,*icpcol_gid;

274:     /* need to copy this to free buffer -- should do this globaly */
275:     PetscMalloc1(num_fine_ghosts, &icpcol_gid);
276:     for (cpid=0; cpid<num_fine_ghosts; cpid++) icpcol_gid[cpid] = (PetscInt)PetscRealPart(cpcol_gid[cpid]);

278:     /* get proc of deleted ghost */
279:     VecSetValues(locState, nloc, lid_gid, lid_parent_gid, INSERT_VALUES);
280:     VecAssemblyBegin(locState);
281:     VecAssemblyEnd(locState);
282:     VecScatterBegin(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
283:       VecScatterEnd(mpimat->Mvctx,locState,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
284:     VecGetArray(mpimat->lvec, &cpcol_sel_gid); /* has pe that owns ghost */
285:     for (cpid=0; cpid<num_fine_ghosts; cpid++) {
286:       PetscInt sgid = (PetscInt)PetscRealPart(cpcol_sel_gid[cpid]);
287:       PetscInt gid  = icpcol_gid[cpid];
288:       if (sgid >= my0 && sgid < Iend) { /* I own this deleted */
289:         PetscInt slid = sgid - my0;
290:         /* id_llist[lidj] = id_llist[lid]; id_llist[lid] = lidj; */ /* insert 'lidj' into head of llist */
291:         PetscCDAppendID(agg_lists, slid, gid);
292:       }
293:     }
294:     VecRestoreArray(mpimat->lvec, &cpcol_sel_gid);
295:     PetscFree(icpcol_gid);
296:   } else if (matB) {
297:     VecRestoreArray(mpimat->lvec, &cpcol_gid);
298:   }

300:   /* cache IS of removed nodes, use 'lid_gid' */
301:   /* for (kk=n=0,ix=my0;kk<nloc;kk++,ix++) { */
302:   /*   if (lid_removed[kk]) lid_gid[n++] = ix; */
303:   /* } */
304:   /* PetscCDSetRemovedIS(agg_lists, comm, n, lid_gid); */

306:   PetscFree(lid_cprowID);
307:   PetscFree(lid_gid);
308:   PetscFree(lid_removed);
309:   if (strict_aggs) {
310:     PetscFree(lid_parent_gid);
311:   }
312:   PetscFree(lid_state);

314:   if (mpimat) {
315:     VecDestroy(&ghostState);
316:   }
317:   VecDestroy(&locState);
318:   return(0);
319: }

321: typedef struct {
322:   int dummy;
323: } MatCoarsen_MIS;
324: /*
325:    MIS coarsen, simple greedy.
326: */
329: static PetscErrorCode MatCoarsenApply_MIS(MatCoarsen coarse)
330: {
331:   /* MatCoarsen_MIS *MIS = (MatCoarsen_MIS*)coarse->; */
333:   Mat            mat = coarse->graph;

337:   if (!coarse->perm) {
338:     IS       perm;
339:     PetscInt n,m;
340:     MPI_Comm comm;
341:     PetscObjectGetComm((PetscObject)mat,&comm);
342:     MatGetLocalSize(mat, &m, &n);
343:     ISCreateStride(comm, m, 0, 1, &perm);
344:     maxIndSetAgg(perm, mat, coarse->strict_aggs, coarse->verbose, &coarse->agg_lists);
345:     ISDestroy(&perm);
346:   } else {
347:     maxIndSetAgg(coarse->perm, mat, coarse->strict_aggs, coarse->verbose, &coarse->agg_lists);
348:   }
349:   return(0);
350: }

354: PetscErrorCode MatCoarsenView_MIS(MatCoarsen coarse,PetscViewer viewer)
355: {
356:   /* MatCoarsen_MIS *MIS = (MatCoarsen_MIS*)coarse->; */
358:   PetscMPIInt    rank;
359:   PetscBool      iascii;

363:   MPI_Comm_rank(PetscObjectComm((PetscObject)coarse),&rank);
364:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
365:   if (iascii) {
366:     PetscViewerASCIISynchronizedPrintf(viewer,"  [%d] MIS aggregator\n",rank);
367:     PetscViewerFlush(viewer);
368:     PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
369:   }
370:   return(0);
371: }

375: PetscErrorCode MatCoarsenDestroy_MIS(MatCoarsen coarse)
376: {
377:   MatCoarsen_MIS *MIS = (MatCoarsen_MIS*)coarse->subctx;

382:   PetscFree(MIS);
383:   return(0);
384: }

386: /*MC
387:    MATCOARSENMIS - Creates a coarsen context via the external package MIS.

389:    Collective on MPI_Comm

391:    Input Parameter:
392: .  coarse - the coarsen context

394:    Options Database Keys:
395: +  -mat_coarsen_MIS_xxx -

397:    Level: beginner

399: .keywords: Coarsen, create, context

401: .seealso: MatCoarsenSetType(), MatCoarsenType

403: M*/

407: PETSC_EXTERN PetscErrorCode MatCoarsenCreate_MIS(MatCoarsen coarse)
408: {
410:   MatCoarsen_MIS *MIS;

413:   PetscNewLog(coarse,&MIS);
414:   coarse->subctx = (void*)MIS;

416:   coarse->ops->apply   = MatCoarsenApply_MIS;
417:   coarse->ops->view    = MatCoarsenView_MIS;
418:   coarse->ops->destroy = MatCoarsenDestroy_MIS;
419:   /* coarse->ops->setfromoptions = MatCoarsenSetFromOptions_MIS; */
420:   return(0);
421: }