Actual source code: jp.c
petsc-dev 2014-02-02
1: #include <petsc-private/matimpl.h> /*I "petscmat.h" I*/
2: #include <petscsf.h>
4: PETSC_EXTERN PetscErrorCode MatColoringLocalColor(MatColoring,PetscSF,PetscSF,PetscReal *,ISColoringValue *,ISColoringValue *);
5: PETSC_EXTERN PetscErrorCode MatColoringDiscoverBoundary(MatColoring,PetscSF,PetscSF,PetscInt *,PetscInt**);
6: PETSC_EXTERN PetscErrorCode MatColoringCreateBipartiteGraph(MatColoring,PetscSF *,PetscSF *);
8: typedef struct {
9: PetscSF etoc;
10: PetscSF etor;
11: PetscReal *wts;
12: PetscReal *wtsinit;
13: PetscReal *wtscol;
14: PetscReal *wtsrow;
15: PetscReal *wtsleafrow;
16: PetscReal *wtsleafcol;
17: PetscReal *wtsspread;
18: ISColoringValue maxcolor;
19: PetscInt statesize;
20: PetscInt stateradix;
21: PetscInt *state;
22: PetscInt *statecol;
23: PetscInt *staterow;
24: PetscInt *stateleafcol;
25: PetscInt *stateleafrow;
26: PetscInt *statespread;
27: ISColoringValue *color;
28: ISColoringValue *mincolor;
29: } MC_JP;
34: PetscErrorCode JPCreateWeights_Private(MatColoring mc)
35: {
36: MC_JP *jp = (MC_JP *)mc->data;
38: PetscInt i,ncols;
39: PetscRandom rand;
40: PetscReal *wts = jp->wts;
41: PetscReal r;
42: const PetscInt *coldegrees;
43: PetscSF etoc=jp->etoc;
46: /* each weight should be the degree plus a random perturbation */
47: PetscSFGetGraph(etoc,&ncols,NULL,NULL,NULL);
48: PetscSFComputeDegreeBegin(etoc,&coldegrees);
49: PetscSFComputeDegreeEnd(etoc,&coldegrees);
50: PetscRandomCreate(PetscObjectComm((PetscObject)mc),&rand);
51: PetscRandomSetFromOptions(rand);
52: for (i=0;i<ncols;i++) {
53: PetscRandomGetValueReal(rand,&r);
54: wts[i] = coldegrees[i] + PetscAbsReal(r);
55: }
56: PetscRandomDestroy(&rand);
57: return(0);
58: }
62: PetscErrorCode JPInitialize_Private(MatColoring mc)
63: {
64: MC_JP *jp = (MC_JP *)mc->data;
65: PetscInt i,croot,cleaf,rroot,rleaf;
69: MatColoringCreateBipartiteGraph(mc,&jp->etoc,&jp->etor);
70: jp->statesize = 1;
71: jp->stateradix = (8*sizeof(PetscInt)-1);
72: PetscSFGetGraph(jp->etoc,&croot,&cleaf,NULL,NULL);
73: PetscSFGetGraph(jp->etor,&rroot,&rleaf,NULL,NULL);
74: PetscMalloc7(croot,&jp->wts,
75: croot,&jp->wtsinit,
76: croot,&jp->wtscol,
77: rroot,&jp->wtsrow,
78: croot,&jp->wtsspread,
79: cleaf,&jp->wtsleafcol,
80: rleaf,&jp->wtsleafrow);
81: PetscMalloc6(croot*jp->statesize,&jp->state,
82: croot*jp->statesize,&jp->statecol,
83: rroot*jp->statesize,&jp->staterow,
84: croot*jp->statesize,&jp->statespread,
85: cleaf*jp->statesize,&jp->stateleafcol,
86: rleaf*jp->statesize,&jp->stateleafrow);
87: PetscMalloc(sizeof(ISColoringValue)*croot,&jp->color);
88: PetscMalloc(sizeof(ISColoringValue)*croot,&jp->mincolor);
89: for (i=0;i<croot;i++) {
90: jp->color[i] = IS_COLORING_MAX;
91: }
92: return(0);
93: }
97: PetscErrorCode MatColoringDestroy_JP(MatColoring mc)
98: {
102: PetscFree(mc->data);
103: return(0);
104: }
108: PetscErrorCode JPTearDown_Private(MatColoring mc)
109: {
110: MC_JP *jp = (MC_JP *)mc->data;
114: PetscSFDestroy(&jp->etoc);
115: PetscSFDestroy(&jp->etor);
116: PetscFree7(jp->wts,
117: jp->wtsinit,
118: jp->wtscol,
119: jp->wtsrow,
120: jp->wtsspread,
121: jp->wtsleafcol,
122: jp->wtsleafrow);
123: PetscFree6(jp->state,
124: jp->statecol,
125: jp->staterow,
126: jp->statespread,
127: jp->stateleafcol,
128: jp->stateleafrow);
129: PetscFree(jp->mincolor);
130: return(0);
131: }
135: PetscErrorCode JPGreatestWeight_Private(MatColoring mc,PetscReal *wtsin,PetscReal *maxwts)
136: {
137: MC_JP *jp = (MC_JP *)mc->data;
138: PetscInt nrows,ncols,nleafrows,nleafcols,nentries,idx,dist=mc->dist;
139: PetscInt i,j,k;
140: const PetscInt *degrees;
141: PetscReal *ewts,*wtsrow=jp->wtsrow,*wtscol=jp->wtscol;
142: PetscSF etoc=jp->etoc,etor=jp->etor;
146: nentries=0;
147: PetscSFGetGraph(etor,&nrows,&nleafrows,NULL,NULL);
148: PetscSFGetGraph(etoc,&ncols,&nleafcols,NULL,NULL);
149: for (i=0;i<ncols;i++) {
150: wtscol[i] = wtsin[i];
151: }
152: for (k=0;k<dist;k++) {
153: if (k%2 == 1) {
154: /* second step takes the row weights to the column weights */
155: PetscSFComputeDegreeBegin(etor,°rees);
156: PetscSFComputeDegreeEnd(etor,°rees);
157: nentries=nleafrows;
158: idx=0;
159: ewts = jp->wtsleafrow;
160: for(i=0;i<nrows;i++) {
161: for (j=0;j<degrees[i];j++) {
162: ewts[idx] = wtsrow[i];
163: idx++;
164: }
165: }
166: for(i=0;i<ncols;i++) {
167: wtscol[i]=0.;
168: }
170: if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
171: PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);
172: PetscSFReduceBegin(etoc,MPI_DOUBLE,ewts,wtscol,MPI_MAX);
173: PetscSFReduceEnd(etoc,MPI_DOUBLE,ewts,wtscol,MPI_MAX);
174: PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);
175: } else {
176: /* first step takes the column weights to the row weights */
177: PetscSFComputeDegreeBegin(etoc,°rees);
178: PetscSFComputeDegreeEnd(etoc,°rees);
179: nentries=nleafcols;
180: ewts = jp->wtsleafcol;
181: idx=0;
182: for(i=0;i<ncols;i++) {
183: for (j=0;j<degrees[i];j++) {
184: ewts[idx] = wtscol[i];
185: idx++;
186: }
187: }
188: for(i=0;i<nrows;i++) {
189: wtsrow[i]=0.;
190: }
191: if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
192: PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);
193: PetscSFReduceBegin(etor,MPI_DOUBLE,ewts,wtsrow,MPI_MAX);
194: PetscSFReduceEnd(etor,MPI_DOUBLE,ewts,wtsrow,MPI_MAX);
195: PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);
196: }
197: }
198: if (mc->dist % 2 == 1) {
199: /* if it's an odd number of steps, copy out the square part */
200: for (i=0;i<ncols;i++) {
201: if (i < nrows) {
202: maxwts[i] = wtsrow[i];
203: } else {
204: maxwts[i] = 0;
205: }
206: }
207: } else {
208: for (i=0;i<ncols;i++) {
209: maxwts[i] = wtscol[i];
210: }
211: }
212: return(0);
213: }
217: PetscErrorCode JPMinColor_Private(MatColoring mc,ISColoringValue *colors,ISColoringValue *mincolor)
218: {
219: MC_JP *jp = (MC_JP *)mc->data;
220: PetscInt nrows,ncols,nleafcols,nleafrows,nentries,idx,dist=mc->dist;
221: PetscInt i,j,k,l,r;
222: const PetscInt *degrees;
223: PetscInt *estate,*mask,mskvalue,*staterow,*statecol;
224: PetscSF etoc=jp->etoc,etor=jp->etor;
225: ISColoringValue curmin;
227: PetscBool minfound;
230: PetscSFGetGraph(etoc,&ncols,&nleafcols,NULL,NULL);
231: PetscSFGetGraph(etor,&nrows,&nleafrows,NULL,NULL);
232: /* reallocate so that we can create new size bitmasks */
233: if (jp->statesize*jp->stateradix <= jp->maxcolor+1) {
234: PetscFree6(jp->state,
235: jp->statecol,
236: jp->staterow,
237: jp->statespread,
238: jp->stateleafcol,
239: jp->stateleafrow);
240: jp->statesize++;
241: PetscMalloc6(ncols*jp->statesize,&jp->state,
242: ncols*jp->statesize,&jp->statecol,
243: nrows*jp->statesize,&jp->staterow,
244: ncols*jp->statesize,&jp->statespread,
245: nleafcols*jp->statesize,&jp->stateleafcol,
246: nleafrows*jp->statesize,&jp->stateleafrow);
247: }
248: statecol = jp->statecol;
249: staterow = jp->staterow;
251: /* set up the bitmask */
252: for (i=0;i<ncols;i++) {
253: if (colors[i] != IS_COLORING_MAX) {
254: r = colors[i] / jp->stateradix;
255: for (j=0;j<jp->statesize;j++) {
256: if (j == r) {
257: statecol[i+j*ncols] = 1;
258: for (l=0;l < colors[i] % jp->stateradix;l++) {
259: statecol[i+j*ncols] *= 2;
260: }
261: } else {
262: statecol[i+j*ncols] = 0;
263: }
264: }
265: } else {
266: for (j=0;j<jp->statesize;j++) {
267: statecol[i+j*ncols] = 0;
268: }
269: }
270: }
272: for (k=0;k<dist;k++) {
273: if (k%2 == 1) {
274: PetscSFComputeDegreeBegin(etor,°rees);
275: PetscSFComputeDegreeEnd(etor,°rees);
276: nentries=0;
277: for(i=0;i<nrows;i++) {
278: nentries += degrees[i];
279: }
280: estate = jp->stateleafrow;
281: for (i=0;i<jp->statesize;i++) {
282: idx=0;
283: for(j=0;j<nrows;j++) {
284: for (l=0;l<degrees[j];l++) {
285: estate[idx] = staterow[j+i*nrows];
286: idx++;
287: }
288: }
289: for (j=0;j<ncols;j++) {
290: statecol[j+i*ncols]=0;
291: }
292: if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
293: PetscLogEventBegin(Mat_Coloring_Comm,etoc,0,0,0);
294: PetscSFReduceBegin(etoc,MPIU_INT,estate,&statecol[i*ncols],MPI_BOR);
295: PetscSFReduceEnd(etoc,MPIU_INT,estate,&statecol[i*ncols],MPI_BOR);
296: PetscLogEventEnd(Mat_Coloring_Comm,etoc,0,0,0);
297: }
298: } else {
299: PetscSFComputeDegreeBegin(etoc,°rees);
300: PetscSFComputeDegreeEnd(etoc,°rees);
301: nentries=0;
302: for(i=0;i<ncols;i++) {
303: nentries += degrees[i];
304: }
305: estate = jp->stateleafcol;
306: for (i=0;i<jp->statesize;i++) {
307: idx=0;
308: for(j=0;j<ncols;j++) {
309: for (l=0;l<degrees[j];l++) {
310: estate[idx] = statecol[j+i*ncols];
311: idx++;
312: }
313: }
314: for (j=0;j<nrows;j++) {
315: staterow[j+i*nrows]=0;
316: }
317: if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
318: PetscLogEventBegin(Mat_Coloring_Comm,etoc,0,0,0);
319: PetscSFReduceBegin(etor,MPIU_INT,estate,&staterow[i*ncols],MPI_BOR);
320: PetscSFReduceEnd(etor,MPIU_INT,estate,&staterow[i*ncols],MPI_BOR);
321: PetscLogEventEnd(Mat_Coloring_Comm,etoc,0,0,0);
322: }
323: }
324: }
325: if (mc->dist % 2 == 1) {
326: mask = staterow;
327: } else {
328: mask = statecol;
329: }
330: /* reconstruct */
331: for (i=0;i<ncols;i++) {
332: curmin = 0;
333: minfound=PETSC_FALSE;
334: for (j=0;j<jp->statesize && !minfound;j++) {
335: mskvalue = mask[i+j*ncols];
336: for (k=0;k<jp->stateradix;k++) {
337: if (mskvalue % 2 == 0) {
338: mincolor[i] = curmin;
339: minfound=PETSC_TRUE;
340: break;
341: }
342: curmin++;
343: mskvalue /= 2;
344: }
345: }
346: if (!minfound) mincolor[i] = (ISColoringValue)jp->stateradix*jp->statesize;
347: }
349: return(0);
350: }
354: PETSC_EXTERN PetscErrorCode MatColoringApply_JP(MatColoring mc,ISColoring *iscoloring)
355: {
356: MC_JP *jp=(MC_JP*)mc->data;
357: PetscErrorCode ierr;
358: PetscInt i,nadded,nadded_total,nadded_total_old,ncolstotal,ncols;
359: PetscInt nr,nc;
360: PetscInt maxcolor_local,maxcolor_global;
361: PetscInt nboundary,*boundary,totalboundary;
362: PetscMPIInt rank;
365: MPI_Comm_rank(PetscObjectComm((PetscObject)mc),&rank);
366: PetscLogEventBegin(Mat_Coloring_SetUp,mc,0,0,0);
367: JPInitialize_Private(mc);
368: PetscLogEventEnd(Mat_Coloring_SetUp,mc,0,0,0);
369: JPCreateWeights_Private(mc);
370: MatGetSize(mc->mat,NULL,&ncolstotal);
371: MatGetLocalSize(mc->mat,NULL,&ncols);
372: PetscSFGetGraph(jp->etor,&nr,NULL,NULL,NULL);
373: PetscSFGetGraph(jp->etoc,&nc,NULL,NULL,NULL);
375: PetscLogEventBegin(Mat_Coloring_Local,mc,0,0,0);
376: MatColoringDiscoverBoundary(mc,jp->etoc,jp->etor,&nboundary,&boundary);
377: PetscLogEventEnd(Mat_Coloring_Local,mc,0,0,0);
378: totalboundary=0;
379: MPI_Allreduce(&nboundary,&totalboundary,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));
380: if (totalboundary > 0) {
381: for (i=0;i<nc;i++) {
382: jp->wtsinit[i] = 0.;
383: jp->state[i]=0;
384: jp->wtsspread[i]=0.;
385: jp->statespread[i]=0;
386: }
387: for (i=0;i<nboundary;i++) {
388: jp->wtsinit[boundary[i]] = jp->wts[boundary[i]];
389: }
390: nadded=0;
391: nadded_total=0;
392: nadded_total_old=0;
393: while (nadded_total < totalboundary) {
394: JPGreatestWeight_Private(mc,jp->wtsinit,jp->wtsspread);
395: JPMinColor_Private(mc,jp->color,jp->mincolor);
396: for (i=0;i<nboundary;i++) {
397: if (jp->wtsinit[boundary[i]] >= jp->wtsspread[boundary[i]] && jp->wtsinit[boundary[i]] > 0.) {
398: /* pick this one */
399: if (mc->maxcolors > jp->mincolor[boundary[i]] || mc->maxcolors==0) {
400: jp->color[boundary[i]] = jp->mincolor[boundary[i]];
401: } else {
402: jp->color[boundary[i]] = mc->maxcolors;
403: }
404: if (jp->color[boundary[i]] > jp->maxcolor) jp->maxcolor = jp->color[boundary[i]];
405: jp->wtsinit[boundary[i]] = 0.;
406: nadded++;
407: }
408: }
409: MPI_Allreduce(&nadded,&nadded_total,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));
410: if (nadded_total == nadded_total_old) {SETERRQ(PetscObjectComm((PetscObject)mc),PETSC_ERR_NOT_CONVERGED,"JP didn't make progress");}
411: nadded_total_old = nadded_total;
412: maxcolor_local = (PetscInt)jp->maxcolor;
413: maxcolor_global = 0;
414: MPI_Allreduce(&maxcolor_local,&maxcolor_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)mc));
415: jp->maxcolor = maxcolor_global;
416: }
417: }
418: PetscLogEventBegin(Mat_Coloring_Local,mc,0,0,0);
419: MatColoringLocalColor(mc,jp->etoc,jp->etor,jp->wts,jp->color,&jp->maxcolor);
420: PetscLogEventEnd(Mat_Coloring_Local,mc,0,0,0);
421: maxcolor_local = (PetscInt)jp->maxcolor;
422: maxcolor_global = 0;
423: MPI_Allreduce(&maxcolor_local,&maxcolor_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)mc));
424: jp->maxcolor = maxcolor_global;
425: PetscLogEventBegin(Mat_Coloring_ISCreate,mc,0,0,0);
426: ISColoringCreate(PetscObjectComm((PetscObject)mc),jp->maxcolor+1,ncols,jp->color,iscoloring);
427: