Actual source code: iscoloring.c
2: #include petscsys.h
3: #include petscis.h
7: /*@C
8: ISColoringDestroy - Destroys a coloring context.
10: Collective on ISColoring
12: Input Parameter:
13: . iscoloring - the coloring context
15: Level: advanced
17: .seealso: ISColoringView(), MatGetColoring()
18: @*/
19: PetscErrorCode ISColoringDestroy(ISColoring iscoloring)
20: {
21: PetscInt i;
26: if (--iscoloring->refct > 0) return(0);
28: if (iscoloring->is) {
29: for (i=0; i<iscoloring->n; i++) {
30: ISDestroy(iscoloring->is[i]);
31: }
32: PetscFree(iscoloring->is);
33: }
34: if (iscoloring->colors) {
35: PetscFree(iscoloring->colors);
36: }
37: PetscCommDestroy(&iscoloring->comm);
38: PetscFree(iscoloring);
39: return(0);
40: }
44: /*@C
45: ISColoringView - Views a coloring context.
47: Collective on ISColoring
49: Input Parameters:
50: + iscoloring - the coloring context
51: - viewer - the viewer
53: Level: advanced
55: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
56: @*/
57: PetscErrorCode ISColoringView(ISColoring iscoloring,PetscViewer viewer)
58: {
59: PetscInt i;
61: PetscTruth iascii;
62: IS *is;
66: if (!viewer) viewer = PETSC_VIEWER_STDOUT_(iscoloring->comm);
69: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
70: if (iascii) {
71: MPI_Comm comm;
72: PetscMPIInt rank;
73: PetscObjectGetComm((PetscObject)viewer,&comm);
74: MPI_Comm_rank(comm,&rank);
75: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
76: PetscViewerFlush(viewer);
77: } else {
78: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
79: }
81: ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
82: for (i=0; i<iscoloring->n; i++) {
83: ISView(iscoloring->is[i],viewer);
84: }
85: ISColoringRestoreIS(iscoloring,&is);
86: return(0);
87: }
91: /*@C
92: ISColoringGetIS - Extracts index sets from the coloring context
94: Collective on ISColoring
96: Input Parameter:
97: . iscoloring - the coloring context
99: Output Parameters:
100: + nn - number of index sets in the coloring context
101: - is - array of index sets
103: Level: advanced
105: .seealso: ISColoringRestoreIS(), ISColoringView()
106: @*/
107: PetscErrorCode ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
108: {
114: if (nn) *nn = iscoloring->n;
115: if (isis) {
116: if (!iscoloring->is) {
117: PetscInt *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
118: ISColoringValue *colors = iscoloring->colors;
119: IS *is;
120:
121: /* generate the lists of nodes for each color */
122: PetscMalloc(nc*sizeof(PetscInt),&mcolors);
123: PetscMemzero(mcolors,nc*sizeof(PetscInt));
124: for (i=0; i<n; i++) {
125: mcolors[colors[i]]++;
126: }
128: PetscMalloc(nc*sizeof(PetscInt*),&ii);
129: PetscMalloc(n*sizeof(PetscInt),&ii[0]);
130: for (i=1; i<nc; i++) {
131: ii[i] = ii[i-1] + mcolors[i-1];
132: }
133:
134: MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
135: base -= iscoloring->N;
136: PetscMemzero(mcolors,nc*sizeof(PetscInt));
137: for (i=0; i<n; i++) {
138: ii[colors[i]][mcolors[colors[i]]++] = i + base;
139: }
140: PetscMalloc(nc*sizeof(IS),&is);
141: for (i=0; i<nc; i++) {
142: ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
143: }
145: iscoloring->is = is;
146: PetscFree(ii[0]);
147: PetscFree(ii);
148: PetscFree(mcolors);
149: }
150: *isis = iscoloring->is;
151: }
153: return(0);
154: }
158: /*@C
159: ISColoringGetIS - Restores the index sets extracted from the coloring context
161: Collective on ISColoring
163: Input Parameter:
164: + iscoloring - the coloring context
165: - is - array of index sets
167: Level: advanced
169: .seealso: ISColoringGetIS(), ISColoringView()
170: @*/
171: PetscErrorCode ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
172: {
175:
176: /* currently nothing is done here */
178: return(0);
179: }
184: /*@C
185: ISColoringCreate - Generates an ISColoring context from lists (provided
186: by each processor) of colors for each node.
188: Collective on MPI_Comm
190: Input Parameters:
191: + comm - communicator for the processors creating the coloring
192: . n - number of nodes on this processor
193: - colors - array containing the colors for this processor, color
194: numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
195: and should NOT be freed (The ISColoringDestroy() will free it).
197: Output Parameter:
198: . iscoloring - the resulting coloring data structure
200: Options Database Key:
201: . -is_coloring_view - Activates ISColoringView()
203: Level: advanced
204:
205: Notes: By default sets coloring type to IS_COLORING_LOCAL
207: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()
209: @*/
210: PetscErrorCode ISColoringCreate(MPI_Comm comm,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
211: {
213: PetscMPIInt size,rank,tag;
214: PetscInt base,top,i;
215: PetscInt nc,ncwork;
216: PetscTruth flg;
217: MPI_Status status;
220: PetscNew(struct _p_ISColoring,iscoloring);
221: PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
222: comm = (*iscoloring)->comm;
224: /* compute the number of the first node on my processor */
225: MPI_Comm_size(comm,&size);
227: /* should use MPI_Scan() */
228: MPI_Comm_rank(comm,&rank);
229: if (!rank) {
230: base = 0;
231: top = n;
232: } else {
233: MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
234: top = base+n;
235: }
236: if (rank < size-1) {
237: MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
238: }
240: /* compute the total number of colors */
241: ncwork = 0;
242: for (i=0; i<n; i++) {
243: if (ncwork < colors[i]) ncwork = colors[i];
244: }
245: ncwork++;
246: MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
247: (*iscoloring)->n = nc;
248: (*iscoloring)->is = 0;
249: (*iscoloring)->colors = (ISColoringValue *)colors;
250: (*iscoloring)->N = n;
251: (*iscoloring)->refct = 1;
252: (*iscoloring)->ctype = IS_COLORING_LOCAL;
254: PetscOptionsHasName(PETSC_NULL,"-is_coloring_view",&flg);
255: if (flg) {
256: ISColoringView(*iscoloring,PETSC_VIEWER_STDOUT_((*iscoloring)->comm));
257: }
258: PetscLogInfo(0,"ISColoringCreate: Number of colors %d\n",nc);
259: return(0);
260: }
264: /*@C
265: ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
266: generates an IS that contains a new global node number for each index based
267: on the partitioing.
269: Collective on IS
271: Input Parameters
272: . partitioning - a partitioning as generated by MatPartitioningApply()
274: Output Parameter:
275: . is - on each processor the index set that defines the global numbers
276: (in the new numbering) for all the nodes currently (before the partitioning)
277: on that processor
279: Level: advanced
281: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()
283: @*/
284: PetscErrorCode ISPartitioningToNumbering(IS part,IS *is)
285: {
286: MPI_Comm comm;
287: PetscInt i,*indices,np,npt,n,*starts,*sums,*lsizes,*newi;
289: PetscMPIInt size;
292: PetscObjectGetComm((PetscObject)part,&comm);
293: MPI_Comm_size(comm,&size);
295: /* count the number of partitions, i.e., virtual processors */
296: ISGetLocalSize(part,&n);
297: ISGetIndices(part,&indices);
298: np = 0;
299: for (i=0; i<n; i++) {
300: np = PetscMax(np,indices[i]);
301: }
302: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
303: np = npt+1; /* so that it looks like a MPI_Comm_size output */
305: /*
306: lsizes - number of elements of each partition on this particular processor
307: sums - total number of "previous" nodes for any particular partition
308: starts - global number of first element in each partition on this processor
309: */
310: PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
311: PetscMemzero(lsizes,np*sizeof(PetscInt));
312: for (i=0; i<n; i++) {
313: lsizes[indices[i]]++;
314: }
315: MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
316: MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
317: for (i=0; i<np; i++) {
318: starts[i] -= lsizes[i];
319: }
320: for (i=1; i<np; i++) {
321: sums[i] += sums[i-1];
322: starts[i] += sums[i-1];
323: }
325: /*
326: For each local index give it the new global number
327: */
328: PetscMalloc(n*sizeof(PetscInt),&newi);
329: for (i=0; i<n; i++) {
330: newi[i] = starts[indices[i]]++;
331: }
332: PetscFree3(lsizes,starts,sums);
334: ISRestoreIndices(part,&indices);
335: ISCreateGeneral(comm,n,newi,is);
336: PetscFree(newi);
337: ISSetPermutation(*is);
338: return(0);
339: }
343: /*@C
344: ISPartitioningCount - Takes a ISPartitioning and determines the number of
345: resulting elements on each processor
347: Collective on IS
349: Input Parameters:
350: . partitioning - a partitioning as generated by MatPartitioningApply()
352: Output Parameter:
353: . count - array of length size, to contain the number of elements assigned
354: to each partition, where size is the number of partitions generated
355: (see notes below).
357: Level: advanced
359: Notes:
360: By default the number of partitions generated (and thus the length
361: of count) is the size of the communicator associated with IS,
362: but it can be set by MatPartitioningSetNParts. The resulting array
363: of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.
366: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
367: MatPartitioningSetNParts()
369: @*/
370: PetscErrorCode ISPartitioningCount(IS part,PetscInt count[])
371: {
372: MPI_Comm comm;
373: PetscInt i,*indices,np,npt,n,*lsizes;
375: PetscMPIInt size;
378: PetscObjectGetComm((PetscObject)part,&comm);
379: MPI_Comm_size(comm,&size);
381: /* count the number of partitions */
382: ISGetLocalSize(part,&n);
383: ISGetIndices(part,&indices);
384: np = 0;
385: for (i=0; i<n; i++) {
386: np = PetscMax(np,indices[i]);
387: }
388: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
389: np = npt+1; /* so that it looks like a MPI_Comm_size output */
391: /*
392: lsizes - number of elements of each partition on this particular processor
393: sums - total number of "previous" nodes for any particular partition
394: starts - global number of first element in each partition on this processor
395: */
396: PetscMalloc(np*sizeof(PetscInt),&lsizes);
397: PetscMemzero(lsizes,np*sizeof(PetscInt));
398: for (i=0; i<n; i++) {
399: lsizes[indices[i]]++;
400: }
401: ISRestoreIndices(part,&indices);
402: MPI_Allreduce(lsizes,count,np,MPIU_INT,MPI_SUM,comm);
403: PetscFree(lsizes);
404: return(0);
405: }
409: /*@C
410: ISAllGather - Given an index set (IS) on each processor, generates a large
411: index set (same on each processor) by concatenating together each
412: processors index set.
414: Collective on IS
416: Input Parameter:
417: . is - the distributed index set
419: Output Parameter:
420: . isout - the concatenated index set (same on all processors)
422: Notes:
423: ISAllGather() is clearly not scalable for large index sets.
425: The IS created on each processor must be created with a common
426: communicator (e.g., PETSC_COMM_WORLD). If the index sets were created
427: with PETSC_COMM_SELF, this routine will not work as expected, since
428: each process will generate its own new IS that consists only of
429: itself.
431: Level: intermediate
433: Concepts: gather^index sets
434: Concepts: index sets^gathering to all processors
435: Concepts: IS^gathering to all processors
437: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
438: @*/
439: PetscErrorCode ISAllGather(IS is,IS *isout)
440: {
442: PetscInt *indices,n,*lindices,i,N;
443: MPI_Comm comm;
444: PetscMPIInt size,*sizes,*offsets,nn;
450: PetscObjectGetComm((PetscObject)is,&comm);
451: MPI_Comm_size(comm,&size);
452: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
453:
454: ISGetLocalSize(is,&n);
455: nn = (PetscMPIInt)n;
456: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
457: offsets[0] = 0;
458: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
459: N = offsets[size-1] + sizes[size-1];
461: PetscMalloc(N*sizeof(PetscInt),&indices);
462: ISGetIndices(is,&lindices);
463: MPI_Allgatherv(lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
464: ISRestoreIndices(is,&lindices);
465: PetscFree(sizes);
467: ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
468: PetscFree2(indices,offsets);
469: return(0);
470: }
474: /*@C
475: ISAllGatherIndices - Given a a set of integers on each processor, generates a large
476: set (same on each processor) by concatenating together each processors integers
478: Collective on MPI_Comm
480: Input Parameter:
481: + comm - communicator to share the indices
482: . n - local size of set
483: - lindices - local indices
485: Output Parameter:
486: + outN - total number of indices
487: - outindices - all of the integers
489: Notes:
490: ISAllGatherIndices() is clearly not scalable for large index sets.
493: Level: intermediate
495: Concepts: gather^index sets
496: Concepts: index sets^gathering to all processors
497: Concepts: IS^gathering to all processors
499: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
500: @*/
501: PetscErrorCode ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
502: {
504: PetscInt *indices,i,N;
505: PetscMPIInt size,*sizes,*offsets,nn;
508: MPI_Comm_size(comm,&size);
509: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
510:
511: nn = n;
512: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
513: offsets[0] = 0;
514: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
515: N = offsets[size-1] + sizes[size-1];
517: PetscMalloc(N*sizeof(PetscInt),&indices);
518: MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
519: PetscFree2(sizes,offsets);
521: *outindices = indices;
522: if (outN) *outN = N;
523: return(0);
524: }
530: /*@C
531: ISAllGatherColors - Given a a set of colors on each processor, generates a large
532: set (same on each processor) by concatenating together each processors colors
534: Collective on MPI_Comm
536: Input Parameter:
537: + comm - communicator to share the indices
538: . n - local size of set
539: - lindices - local colors
541: Output Parameter:
542: + outN - total number of indices
543: - outindices - all of the colors
545: Notes:
546: ISAllGatherColors() is clearly not scalable for large index sets.
549: Level: intermediate
551: Concepts: gather^index sets
552: Concepts: index sets^gathering to all processors
553: Concepts: IS^gathering to all processors
555: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
556: @*/
557: PetscErrorCode ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
558: {
559: ISColoringValue *indices;
560: PetscErrorCode ierr;
561: PetscInt i,N;
562: PetscMPIInt size,*offsets,*sizes, nn = n;
565: MPI_Comm_size(comm,&size);
566: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
567:
568: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
569: offsets[0] = 0;
570: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
571: N = offsets[size-1] + sizes[size-1];
572: PetscFree2(sizes,offsets);
574: PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
575: MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);
577: *outindices = indices;
578: if (outN) *outN = N;
579: return(0);
580: }