Actual source code: jp.c

petsc-dev 2014-02-02
Report Typos and Errors
  1: #include <petsc-private/matimpl.h>      /*I "petscmat.h"  I*/
  2: #include <petscsf.h>

  4: PETSC_EXTERN PetscErrorCode MatColoringLocalColor(MatColoring,PetscSF,PetscSF,PetscReal *,ISColoringValue *,ISColoringValue *);
  5: PETSC_EXTERN PetscErrorCode MatColoringDiscoverBoundary(MatColoring,PetscSF,PetscSF,PetscInt *,PetscInt**);
  6: PETSC_EXTERN PetscErrorCode MatColoringCreateBipartiteGraph(MatColoring,PetscSF *,PetscSF *);

  8: typedef struct {
  9:   PetscSF         etoc;
 10:   PetscSF         etor;
 11:   PetscReal       *wts;
 12:   PetscReal       *wtsinit;
 13:   PetscReal       *wtscol;
 14:   PetscReal       *wtsrow;
 15:   PetscReal       *wtsleafrow;
 16:   PetscReal       *wtsleafcol;
 17:   PetscReal       *wtsspread;
 18:   ISColoringValue maxcolor;
 19:   PetscInt        statesize;
 20:   PetscInt        stateradix;
 21:   PetscInt        *state;
 22:   PetscInt        *statecol;
 23:   PetscInt        *staterow;
 24:   PetscInt        *stateleafcol;
 25:   PetscInt        *stateleafrow;
 26:   PetscInt        *statespread;
 27:   ISColoringValue *color;
 28:   ISColoringValue *mincolor;
 29: } MC_JP;


 34: PetscErrorCode JPCreateWeights_Private(MatColoring mc)
 35: {
 36:   MC_JP          *jp = (MC_JP *)mc->data;
 38:   PetscInt       i,ncols;
 39:   PetscRandom    rand;
 40:   PetscReal      *wts = jp->wts;
 41:   PetscReal      r;
 42:   const PetscInt *coldegrees;
 43:   PetscSF        etoc=jp->etoc;

 46:   /* each weight should be the degree plus a random perturbation */
 47:   PetscSFGetGraph(etoc,&ncols,NULL,NULL,NULL);
 48:   PetscSFComputeDegreeBegin(etoc,&coldegrees);
 49:   PetscSFComputeDegreeEnd(etoc,&coldegrees);
 50:   PetscRandomCreate(PetscObjectComm((PetscObject)mc),&rand);
 51:   PetscRandomSetFromOptions(rand);
 52:   for (i=0;i<ncols;i++) {
 53:     PetscRandomGetValueReal(rand,&r);
 54:     wts[i] = coldegrees[i] + PetscAbsReal(r);
 55:   }
 56:   PetscRandomDestroy(&rand);
 57:   return(0);
 58: }

 62: PetscErrorCode JPInitialize_Private(MatColoring mc)
 63: {
 64:   MC_JP          *jp = (MC_JP *)mc->data;
 65:   PetscInt       i,croot,cleaf,rroot,rleaf;

 69:   MatColoringCreateBipartiteGraph(mc,&jp->etoc,&jp->etor);
 70:   jp->statesize = 1;
 71:   jp->stateradix = (8*sizeof(PetscInt)-1);
 72:   PetscSFGetGraph(jp->etoc,&croot,&cleaf,NULL,NULL);
 73:   PetscSFGetGraph(jp->etor,&rroot,&rleaf,NULL,NULL);
 74:   PetscMalloc7(croot,&jp->wts,
 75:                       croot,&jp->wtsinit,
 76:                       croot,&jp->wtscol,
 77:                       rroot,&jp->wtsrow,
 78:                       croot,&jp->wtsspread,
 79:                       cleaf,&jp->wtsleafcol,
 80:                       rleaf,&jp->wtsleafrow);
 81:   PetscMalloc6(croot*jp->statesize,&jp->state,
 82:                       croot*jp->statesize,&jp->statecol,
 83:                       rroot*jp->statesize,&jp->staterow,
 84:                       croot*jp->statesize,&jp->statespread,
 85:                       cleaf*jp->statesize,&jp->stateleafcol,
 86:                       rleaf*jp->statesize,&jp->stateleafrow);
 87:   PetscMalloc(sizeof(ISColoringValue)*croot,&jp->color);
 88:   PetscMalloc(sizeof(ISColoringValue)*croot,&jp->mincolor);
 89:   for (i=0;i<croot;i++) {
 90:     jp->color[i] = IS_COLORING_MAX;
 91:   }
 92:   return(0);
 93: }

 97: PetscErrorCode MatColoringDestroy_JP(MatColoring mc)
 98: {

102:   PetscFree(mc->data);
103:   return(0);
104: }

108: PetscErrorCode JPTearDown_Private(MatColoring mc)
109: {
110:   MC_JP          *jp = (MC_JP *)mc->data;

114:   PetscSFDestroy(&jp->etoc);
115:   PetscSFDestroy(&jp->etor);
116:   PetscFree7(jp->wts,
117:                     jp->wtsinit,
118:                     jp->wtscol,
119:                     jp->wtsrow,
120:                     jp->wtsspread,
121:                     jp->wtsleafcol,
122:                     jp->wtsleafrow);
123:   PetscFree6(jp->state,
124:                     jp->statecol,
125:                     jp->staterow,
126:                     jp->statespread,
127:                     jp->stateleafcol,
128:                     jp->stateleafrow);
129:   PetscFree(jp->mincolor);
130:   return(0);
131: }

135: PetscErrorCode JPGreatestWeight_Private(MatColoring mc,PetscReal *wtsin,PetscReal *maxwts)
136: {
137:   MC_JP         *jp = (MC_JP *)mc->data;
138:   PetscInt       nrows,ncols,nleafrows,nleafcols,nentries,idx,dist=mc->dist;
139:   PetscInt       i,j,k;
140:   const PetscInt *degrees;
141:   PetscReal      *ewts,*wtsrow=jp->wtsrow,*wtscol=jp->wtscol;
142:   PetscSF        etoc=jp->etoc,etor=jp->etor;

146:   nentries=0;
147:   PetscSFGetGraph(etor,&nrows,&nleafrows,NULL,NULL);
148:   PetscSFGetGraph(etoc,&ncols,&nleafcols,NULL,NULL);
149:   for (i=0;i<ncols;i++) {
150:     wtscol[i] = wtsin[i];
151:   }
152:   for (k=0;k<dist;k++) {
153:     if (k%2 == 1) {
154:       /* second step takes the row weights to the column weights */
155:       PetscSFComputeDegreeBegin(etor,&degrees);
156:       PetscSFComputeDegreeEnd(etor,&degrees);
157:       nentries=nleafrows;
158:       idx=0;
159:       ewts = jp->wtsleafrow;
160:       for(i=0;i<nrows;i++) {
161:         for (j=0;j<degrees[i];j++) {
162:           ewts[idx] = wtsrow[i];
163:           idx++;
164:         }
165:       }
166:       for(i=0;i<ncols;i++) {
167:         wtscol[i]=0.;
168:       }

170:       if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
171:       PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);
172:       PetscSFReduceBegin(etoc,MPI_DOUBLE,ewts,wtscol,MPI_MAX);
173:       PetscSFReduceEnd(etoc,MPI_DOUBLE,ewts,wtscol,MPI_MAX);
174:       PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);
175:     } else {
176:       /* first step takes the column weights to the row weights */
177:       PetscSFComputeDegreeBegin(etoc,&degrees);
178:       PetscSFComputeDegreeEnd(etoc,&degrees);
179:       nentries=nleafcols;
180:       ewts = jp->wtsleafcol;
181:       idx=0;
182:       for(i=0;i<ncols;i++) {
183:         for (j=0;j<degrees[i];j++) {
184:           ewts[idx] = wtscol[i];
185:           idx++;
186:         }
187:       }
188:       for(i=0;i<nrows;i++) {
189:         wtsrow[i]=0.;
190:       }
191:       if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
192:       PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);
193:       PetscSFReduceBegin(etor,MPI_DOUBLE,ewts,wtsrow,MPI_MAX);
194:       PetscSFReduceEnd(etor,MPI_DOUBLE,ewts,wtsrow,MPI_MAX);
195:       PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);
196:     }
197:   }
198:   if (mc->dist % 2 == 1) {
199:     /* if it's an odd number of steps, copy out the square part */
200:     for (i=0;i<ncols;i++) {
201:       if (i < nrows) {
202:         maxwts[i] = wtsrow[i];
203:       } else {
204:         maxwts[i] = 0;
205:       }
206:     }
207:   } else {
208:     for (i=0;i<ncols;i++) {
209:       maxwts[i] = wtscol[i];
210:     }
211:   }
212:   return(0);
213: }

217: PetscErrorCode JPMinColor_Private(MatColoring mc,ISColoringValue *colors,ISColoringValue *mincolor)
218: {
219:   MC_JP          *jp = (MC_JP *)mc->data;
220:   PetscInt       nrows,ncols,nleafcols,nleafrows,nentries,idx,dist=mc->dist;
221:   PetscInt       i,j,k,l,r;
222:   const PetscInt *degrees;
223:   PetscInt       *estate,*mask,mskvalue,*staterow,*statecol;
224:   PetscSF        etoc=jp->etoc,etor=jp->etor;
225:   ISColoringValue curmin;
227:   PetscBool      minfound;

230:   PetscSFGetGraph(etoc,&ncols,&nleafcols,NULL,NULL);
231:   PetscSFGetGraph(etor,&nrows,&nleafrows,NULL,NULL);
232:   /* reallocate so that we can create new size bitmasks */
233:   if (jp->statesize*jp->stateradix <= jp->maxcolor+1) {
234:     PetscFree6(jp->state,
235:                       jp->statecol,
236:                       jp->staterow,
237:                       jp->statespread,
238:                       jp->stateleafcol,
239:                       jp->stateleafrow);
240:     jp->statesize++;
241:     PetscMalloc6(ncols*jp->statesize,&jp->state,
242:                         ncols*jp->statesize,&jp->statecol,
243:                         nrows*jp->statesize,&jp->staterow,
244:                         ncols*jp->statesize,&jp->statespread,
245:                         nleafcols*jp->statesize,&jp->stateleafcol,
246:                         nleafrows*jp->statesize,&jp->stateleafrow);
247:   }
248:   statecol = jp->statecol;
249:   staterow = jp->staterow;

251:   /* set up the bitmask */
252:   for (i=0;i<ncols;i++) {
253:     if (colors[i] != IS_COLORING_MAX) {
254:       r = colors[i] / jp->stateradix;
255:       for (j=0;j<jp->statesize;j++) {
256:         if (j == r) {
257:           statecol[i+j*ncols] = 1;
258:           for (l=0;l < colors[i] % jp->stateradix;l++) {
259:             statecol[i+j*ncols] *= 2;
260:           }
261:         } else {
262:           statecol[i+j*ncols] = 0;
263:         }
264:       }
265:     } else {
266:       for (j=0;j<jp->statesize;j++) {
267:         statecol[i+j*ncols] = 0;
268:       }
269:     }
270:   }

272:   for (k=0;k<dist;k++) {
273:     if (k%2 == 1) {
274:       PetscSFComputeDegreeBegin(etor,&degrees);
275:       PetscSFComputeDegreeEnd(etor,&degrees);
276:       nentries=0;
277:       for(i=0;i<nrows;i++) {
278:         nentries += degrees[i];
279:       }
280:       estate = jp->stateleafrow;
281:       for (i=0;i<jp->statesize;i++) {
282:         idx=0;
283:         for(j=0;j<nrows;j++) {
284:           for (l=0;l<degrees[j];l++) {
285:             estate[idx] = staterow[j+i*nrows];
286:             idx++;
287:           }
288:         }
289:         for (j=0;j<ncols;j++) {
290:           statecol[j+i*ncols]=0;
291:         }
292:         if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
293:         PetscLogEventBegin(Mat_Coloring_Comm,etoc,0,0,0);
294:         PetscSFReduceBegin(etoc,MPIU_INT,estate,&statecol[i*ncols],MPI_BOR);
295:         PetscSFReduceEnd(etoc,MPIU_INT,estate,&statecol[i*ncols],MPI_BOR);
296:         PetscLogEventEnd(Mat_Coloring_Comm,etoc,0,0,0);
297:       }
298:     } else {
299:       PetscSFComputeDegreeBegin(etoc,&degrees);
300:       PetscSFComputeDegreeEnd(etoc,&degrees);
301:       nentries=0;
302:       for(i=0;i<ncols;i++) {
303:         nentries += degrees[i];
304:       }
305:       estate = jp->stateleafcol;
306:       for (i=0;i<jp->statesize;i++) {
307:         idx=0;
308:         for(j=0;j<ncols;j++) {
309:           for (l=0;l<degrees[j];l++) {
310:             estate[idx] = statecol[j+i*ncols];
311:             idx++;
312:           }
313:         }
314:         for (j=0;j<nrows;j++) {
315:           staterow[j+i*nrows]=0;
316:         }
317:         if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
318:         PetscLogEventBegin(Mat_Coloring_Comm,etoc,0,0,0);
319:         PetscSFReduceBegin(etor,MPIU_INT,estate,&staterow[i*ncols],MPI_BOR);
320:         PetscSFReduceEnd(etor,MPIU_INT,estate,&staterow[i*ncols],MPI_BOR);
321:         PetscLogEventEnd(Mat_Coloring_Comm,etoc,0,0,0);
322:       }
323:     }
324:   }
325:   if (mc->dist % 2 == 1) {
326:     mask = staterow;
327:   } else {
328:     mask = statecol;
329:   }
330:   /* reconstruct */
331:   for (i=0;i<ncols;i++) {
332:     curmin = 0;
333:     minfound=PETSC_FALSE;
334:     for (j=0;j<jp->statesize && !minfound;j++) {
335:       mskvalue = mask[i+j*ncols];
336:       for (k=0;k<jp->stateradix;k++) {
337:         if (mskvalue % 2 == 0) {
338:           mincolor[i] = curmin;
339:           minfound=PETSC_TRUE;
340:           break;
341:         }
342:         curmin++;
343:         mskvalue /= 2;
344:       }
345:     }
346:     if (!minfound) mincolor[i] = (ISColoringValue)jp->stateradix*jp->statesize;
347:   }

349:   return(0);
350: }

354: PETSC_EXTERN PetscErrorCode MatColoringApply_JP(MatColoring mc,ISColoring *iscoloring)
355: {
356:   MC_JP           *jp=(MC_JP*)mc->data;
357:   PetscErrorCode  ierr;
358:   PetscInt        i,nadded,nadded_total,nadded_total_old,ncolstotal,ncols;
359:   PetscInt        nr,nc;
360:   PetscInt        maxcolor_local,maxcolor_global;
361:   PetscInt        nboundary,*boundary,totalboundary;
362:   PetscMPIInt     rank;

365:   MPI_Comm_rank(PetscObjectComm((PetscObject)mc),&rank);
366:   PetscLogEventBegin(Mat_Coloring_SetUp,mc,0,0,0);
367:   JPInitialize_Private(mc);
368:   PetscLogEventEnd(Mat_Coloring_SetUp,mc,0,0,0);
369:   JPCreateWeights_Private(mc);
370:   MatGetSize(mc->mat,NULL,&ncolstotal);
371:   MatGetLocalSize(mc->mat,NULL,&ncols);
372:   PetscSFGetGraph(jp->etor,&nr,NULL,NULL,NULL);
373:   PetscSFGetGraph(jp->etoc,&nc,NULL,NULL,NULL);

375:   PetscLogEventBegin(Mat_Coloring_Local,mc,0,0,0);
376:   MatColoringDiscoverBoundary(mc,jp->etoc,jp->etor,&nboundary,&boundary);
377:   PetscLogEventEnd(Mat_Coloring_Local,mc,0,0,0);
378:   totalboundary=0;
379:   MPI_Allreduce(&nboundary,&totalboundary,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));
380:   if (totalboundary > 0) {
381:     for (i=0;i<nc;i++) {
382:       jp->wtsinit[i] = 0.;
383:       jp->state[i]=0;
384:       jp->wtsspread[i]=0.;
385:       jp->statespread[i]=0;
386:     }
387:     for (i=0;i<nboundary;i++) {
388:       jp->wtsinit[boundary[i]] = jp->wts[boundary[i]];
389:     }
390:     nadded=0;
391:     nadded_total=0;
392:     nadded_total_old=0;
393:     while (nadded_total < totalboundary) {
394:       JPGreatestWeight_Private(mc,jp->wtsinit,jp->wtsspread);
395:       JPMinColor_Private(mc,jp->color,jp->mincolor);
396:       for (i=0;i<nboundary;i++) {
397:         if (jp->wtsinit[boundary[i]] >= jp->wtsspread[boundary[i]] && jp->wtsinit[boundary[i]] > 0.) {
398:           /* pick this one */
399:           if (mc->maxcolors > jp->mincolor[boundary[i]] || mc->maxcolors==0) {
400:             jp->color[boundary[i]] = jp->mincolor[boundary[i]];
401:           } else {
402:             jp->color[boundary[i]] = mc->maxcolors;
403:           }
404:           if (jp->color[boundary[i]] > jp->maxcolor) jp->maxcolor = jp->color[boundary[i]];
405:           jp->wtsinit[boundary[i]] = 0.;
406:           nadded++;
407:         }
408:       }
409:       MPI_Allreduce(&nadded,&nadded_total,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));
410:       if (nadded_total == nadded_total_old) {SETERRQ(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"JP didn't make progress");}
411:       nadded_total_old = nadded_total;
412:       maxcolor_local = (PetscInt)jp->maxcolor;
413:       maxcolor_global = 0;
414:       MPI_Allreduce(&maxcolor_local,&maxcolor_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)mc));
415:       jp->maxcolor = maxcolor_global;
416:     }
417:   }
418:   PetscLogEventBegin(Mat_Coloring_Local,mc,0,0,0);
419:   MatColoringLocalColor(mc,jp->etoc,jp->etor,jp->wts,jp->color,&jp->maxcolor);
420:   PetscLogEventEnd(Mat_Coloring_Local,mc,0,0,0);
421:   maxcolor_local = (PetscInt)jp->maxcolor;
422:   maxcolor_global = 0;
423:   MPI_Allreduce(&maxcolor_local,&maxcolor_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)mc));
424:   jp->maxcolor = maxcolor_global;
425:   PetscLogEventBegin(Mat_Coloring_ISCreate,mc,0,0,0);
426:   ISColoringCreate(PetscObjectComm((PetscObject)mc),jp->maxcolor+1,ncols,jp->color,iscoloring);
427: