Actual source code: iscoloring.c


  2: #include <petsc/private/isimpl.h>
  3: #include <petscviewer.h>
  4: #include <petscsf.h>

  6: const char *const ISColoringTypes[] = {"global", "ghosted", "ISColoringType", "IS_COLORING_", NULL};

  8: PetscErrorCode ISColoringReference(ISColoring coloring)
  9: {
 10:   coloring->refct++;
 11:   return 0;
 12: }

 14: /*@C

 16:     ISColoringSetType - indicates if the coloring is for the local representation (including ghost points) or the global representation

 18:    Collective on coloring

 20:    Input Parameters:
 21: +    coloring - the coloring object
 22: -    type - either IS_COLORING_LOCAL or IS_COLORING_GLOBAL

 24:    Notes:
 25:      With IS_COLORING_LOCAL the coloring is in the numbering of the local vector, for IS_COLORING_GLOBAL it is in the number of the global vector

 27:    Level: intermediate

 29: .seealso: `MatFDColoringCreate()`, `ISColoring`, `ISColoringCreate()`, `IS_COLORING_LOCAL`, `IS_COLORING_GLOBAL`, `ISColoringGetType()`

 31: @*/
 32: PetscErrorCode ISColoringSetType(ISColoring coloring, ISColoringType type)
 33: {
 34:   coloring->ctype = type;
 35:   return 0;
 36: }

 38: /*@C

 40:     ISColoringGetType - gets if the coloring is for the local representation (including ghost points) or the global representation

 42:    Collective on coloring

 44:    Input Parameter:
 45: .   coloring - the coloring object

 47:    Output Parameter:
 48: .    type - either IS_COLORING_LOCAL or IS_COLORING_GLOBAL

 50:    Level: intermediate

 52: .seealso: `MatFDColoringCreate()`, `ISColoring`, `ISColoringCreate()`, `IS_COLORING_LOCAL`, `IS_COLORING_GLOBAL`, `ISColoringSetType()`

 54: @*/
 55: PetscErrorCode ISColoringGetType(ISColoring coloring, ISColoringType *type)
 56: {
 57:   *type = coloring->ctype;
 58:   return 0;
 59: }

 61: /*@
 62:    ISColoringDestroy - Destroys a coloring context.

 64:    Collective on ISColoring

 66:    Input Parameter:
 67: .  iscoloring - the coloring context

 69:    Level: advanced

 71: .seealso: `ISColoringView()`, `MatColoring`
 72: @*/
 73: PetscErrorCode ISColoringDestroy(ISColoring *iscoloring)
 74: {
 75:   PetscInt i;

 77:   if (!*iscoloring) return 0;
 79:   if (--(*iscoloring)->refct > 0) {
 80:     *iscoloring = NULL;
 81:     return 0;
 82:   }

 84:   if ((*iscoloring)->is) {
 85:     for (i = 0; i < (*iscoloring)->n; i++) ISDestroy(&(*iscoloring)->is[i]);
 86:     PetscFree((*iscoloring)->is);
 87:   }
 88:   if ((*iscoloring)->allocated) PetscFree((*iscoloring)->colors);
 89:   PetscCommDestroy(&(*iscoloring)->comm);
 90:   PetscFree((*iscoloring));
 91:   return 0;
 92: }

 94: /*
 95:   ISColoringViewFromOptions - Processes command line options to determine if/how an ISColoring object is to be viewed.

 97:   Collective on ISColoring

 99:   Input Parameters:
100: + obj   - the ISColoring object
101: . prefix - prefix to use for viewing, or NULL to use prefix of 'mat'
102: - optionname - option to activate viewing

104:   Level: intermediate

106:   Developer Note: This cannot use PetscObjectViewFromOptions() because ISColoring is not a PetscObject

108: */
109: PetscErrorCode ISColoringViewFromOptions(ISColoring obj, PetscObject bobj, const char optionname[])
110: {
111:   PetscViewer       viewer;
112:   PetscBool         flg;
113:   PetscViewerFormat format;
114:   char             *prefix;

116:   prefix = bobj ? bobj->prefix : NULL;
117:   PetscOptionsGetViewer(obj->comm, NULL, prefix, optionname, &viewer, &format, &flg);
118:   if (flg) {
119:     PetscViewerPushFormat(viewer, format);
120:     ISColoringView(obj, viewer);
121:     PetscViewerPopFormat(viewer);
122:     PetscViewerDestroy(&viewer);
123:   }
124:   return 0;
125: }

127: /*@C
128:    ISColoringView - Views a coloring context.

130:    Collective on ISColoring

132:    Input Parameters:
133: +  iscoloring - the coloring context
134: -  viewer - the viewer

136:    Level: advanced

138: .seealso: `ISColoringDestroy()`, `ISColoringGetIS()`, `MatColoring`
139: @*/
140: PetscErrorCode ISColoringView(ISColoring iscoloring, PetscViewer viewer)
141: {
142:   PetscInt  i;
143:   PetscBool iascii;
144:   IS       *is;

147:   if (!viewer) PetscViewerASCIIGetStdout(iscoloring->comm, &viewer);

150:   PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii);
151:   if (iascii) {
152:     MPI_Comm    comm;
153:     PetscMPIInt size, rank;

155:     PetscObjectGetComm((PetscObject)viewer, &comm);
156:     MPI_Comm_size(comm, &size);
157:     MPI_Comm_rank(comm, &rank);
158:     PetscViewerASCIIPrintf(viewer, "ISColoring Object: %d MPI processes\n", size);
159:     PetscViewerASCIIPrintf(viewer, "ISColoringType: %s\n", ISColoringTypes[iscoloring->ctype]);
160:     PetscViewerASCIIPushSynchronized(viewer);
161:     PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Number of colors %" PetscInt_FMT "\n", rank, iscoloring->n);
162:     PetscViewerFlush(viewer);
163:     PetscViewerASCIIPopSynchronized(viewer);
164:   }

166:   ISColoringGetIS(iscoloring, PETSC_USE_POINTER, PETSC_IGNORE, &is);
167:   for (i = 0; i < iscoloring->n; i++) ISView(iscoloring->is[i], viewer);
168:   ISColoringRestoreIS(iscoloring, PETSC_USE_POINTER, &is);
169:   return 0;
170: }

172: /*@C
173:    ISColoringGetColors - Returns an array with the color for each node

175:    Not Collective

177:    Input Parameter:
178: .  iscoloring - the coloring context

180:    Output Parameters:
181: +  n - number of nodes
182: .  nc - number of colors
183: -  colors - color for each node

185:    Level: advanced

187: .seealso: `ISColoringRestoreIS()`, `ISColoringView()`, `ISColoringGetIS()`
188: @*/
189: PetscErrorCode ISColoringGetColors(ISColoring iscoloring, PetscInt *n, PetscInt *nc, const ISColoringValue **colors)
190: {

193:   if (n) *n = iscoloring->N;
194:   if (nc) *nc = iscoloring->n;
195:   if (colors) *colors = iscoloring->colors;
196:   return 0;
197: }

199: /*@C
200:    ISColoringGetIS - Extracts index sets from the coloring context. Each is contains the nodes of one color

202:    Collective on ISColoring

204:    Input Parameters:
205: +  iscoloring - the coloring context
206: -  mode - if this value is PETSC_OWN_POINTER then the caller owns the pointer and must free the array of IS and each IS in the array

208:    Output Parameters:
209: +  nn - number of index sets in the coloring context
210: -  is - array of index sets

212:    Level: advanced

214: .seealso: `ISColoringRestoreIS()`, `ISColoringView()`, `ISColoringGetColoring()`
215: @*/
216: PetscErrorCode ISColoringGetIS(ISColoring iscoloring, PetscCopyMode mode, PetscInt *nn, IS *isis[])
217: {

220:   if (nn) *nn = iscoloring->n;
221:   if (isis) {
222:     if (!iscoloring->is) {
223:       PetscInt        *mcolors, **ii, nc = iscoloring->n, i, base, n = iscoloring->N;
224:       ISColoringValue *colors = iscoloring->colors;
225:       IS              *is;

227:       if (PetscDefined(USE_DEBUG)) {
229:       }

231:       /* generate the lists of nodes for each color */
232:       PetscCalloc1(nc, &mcolors);
233:       for (i = 0; i < n; i++) mcolors[colors[i]]++;

235:       PetscMalloc1(nc, &ii);
236:       PetscMalloc1(n, &ii[0]);
237:       for (i = 1; i < nc; i++) ii[i] = ii[i - 1] + mcolors[i - 1];
238:       PetscArrayzero(mcolors, nc);

240:       if (iscoloring->ctype == IS_COLORING_GLOBAL) {
241:         MPI_Scan(&iscoloring->N, &base, 1, MPIU_INT, MPI_SUM, iscoloring->comm);
242:         base -= iscoloring->N;
243:         for (i = 0; i < n; i++) ii[colors[i]][mcolors[colors[i]]++] = i + base; /* global idx */
244:       } else if (iscoloring->ctype == IS_COLORING_LOCAL) {
245:         for (i = 0; i < n; i++) ii[colors[i]][mcolors[colors[i]]++] = i; /* local idx */
246:       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Not provided for this ISColoringType type");

248:       PetscMalloc1(nc, &is);
249:       for (i = 0; i < nc; i++) ISCreateGeneral(iscoloring->comm, mcolors[i], ii[i], PETSC_COPY_VALUES, is + i);

251:       if (mode != PETSC_OWN_POINTER) iscoloring->is = is;
252:       *isis = is;
253:       PetscFree(ii[0]);
254:       PetscFree(ii);
255:       PetscFree(mcolors);
256:     } else {
257:       *isis = iscoloring->is;
258:     }
259:   }
260:   return 0;
261: }

263: /*@C
264:    ISColoringRestoreIS - Restores the index sets extracted from the coloring context

266:    Collective on ISColoring

268:    Input Parameters:
269: +  iscoloring - the coloring context
270: .  mode - who retains ownership of the is
271: -  is - array of index sets

273:    Level: advanced

275: .seealso: `ISColoringGetIS()`, `ISColoringView()`
276: @*/
277: PetscErrorCode ISColoringRestoreIS(ISColoring iscoloring, PetscCopyMode mode, IS *is[])
278: {

281:   /* currently nothing is done here */
282:   return 0;
283: }

285: /*@
286:     ISColoringCreate - Generates an ISColoring context from lists (provided
287:     by each processor) of colors for each node.

289:     Collective

291:     Input Parameters:
292: +   comm - communicator for the processors creating the coloring
293: .   ncolors - max color value
294: .   n - number of nodes on this processor
295: .   colors - array containing the colors for this processor, color numbers begin at 0.
296: -   mode - see PetscCopyMode for meaning of this flag.

298:     Output Parameter:
299: .   iscoloring - the resulting coloring data structure

301:     Options Database Key:
302: .   -is_coloring_view - Activates ISColoringView()

304:    Level: advanced

306:     Notes:
307:     By default sets coloring type to  IS_COLORING_GLOBAL

309: .seealso: `MatColoringCreate()`, `ISColoringView()`, `ISColoringDestroy()`, `ISColoringSetType()`

311: @*/
312: PetscErrorCode ISColoringCreate(MPI_Comm comm, PetscInt ncolors, PetscInt n, const ISColoringValue colors[], PetscCopyMode mode, ISColoring *iscoloring)
313: {
314:   PetscMPIInt size, rank, tag;
315:   PetscInt    base, top, i;
316:   PetscInt    nc, ncwork;
317:   MPI_Status  status;

319:   if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
321:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Max color value exceeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short?\n Current max: %d user requested: %" PetscInt_FMT, PETSC_IS_COLORING_MAX, ncolors);
322:   }
323:   PetscNew(iscoloring);
324:   PetscCommDuplicate(comm, &(*iscoloring)->comm, &tag);
325:   comm = (*iscoloring)->comm;

327:   /* compute the number of the first node on my processor */
328:   MPI_Comm_size(comm, &size);

330:   /* should use MPI_Scan() */
331:   MPI_Comm_rank(comm, &rank);
332:   if (rank == 0) {
333:     base = 0;
334:     top  = n;
335:   } else {
336:     MPI_Recv(&base, 1, MPIU_INT, rank - 1, tag, comm, &status);
337:     top = base + n;
338:   }
339:   if (rank < size - 1) MPI_Send(&top, 1, MPIU_INT, rank + 1, tag, comm);

341:   /* compute the total number of colors */
342:   ncwork = 0;
343:   for (i = 0; i < n; i++) {
344:     if (ncwork < colors[i]) ncwork = colors[i];
345:   }
346:   ncwork++;
347:   MPIU_Allreduce(&ncwork, &nc, 1, MPIU_INT, MPI_MAX, comm);
349:   (*iscoloring)->n     = nc;
350:   (*iscoloring)->is    = NULL;
351:   (*iscoloring)->N     = n;
352:   (*iscoloring)->refct = 1;
353:   (*iscoloring)->ctype = IS_COLORING_GLOBAL;
354:   if (mode == PETSC_COPY_VALUES) {
355:     PetscMalloc1(n, &(*iscoloring)->colors);
356:     PetscArraycpy((*iscoloring)->colors, colors, n);
357:     (*iscoloring)->allocated = PETSC_TRUE;
358:   } else if (mode == PETSC_OWN_POINTER) {
359:     (*iscoloring)->colors    = (ISColoringValue *)colors;
360:     (*iscoloring)->allocated = PETSC_TRUE;
361:   } else {
362:     (*iscoloring)->colors    = (ISColoringValue *)colors;
363:     (*iscoloring)->allocated = PETSC_FALSE;
364:   }
365:   ISColoringViewFromOptions(*iscoloring, NULL, "-is_coloring_view");
366:   PetscInfo(0, "Number of colors %" PetscInt_FMT "\n", nc);
367:   return 0;
368: }

370: /*@
371:     ISBuildTwoSided - Takes an IS that describes where we will go. Generates an IS that contains new numbers from remote or local
372:     on the IS.

374:     Collective on IS

376:     Input Parameters:
377: +   ito - an IS describes where we will go. Negative target rank will be ignored
378: -   toindx - an IS describes what indices should send. NULL means sending natural numbering

380:     Output Parameter:
381: .   rows - contains new numbers from remote or local

383:    Level: advanced

385: .seealso: `MatPartitioningCreate()`, `ISPartitioningToNumbering()`, `ISPartitioningCount()`

387: @*/
388: PetscErrorCode ISBuildTwoSided(IS ito, IS toindx, IS *rows)
389: {
390:   const PetscInt *ito_indices, *toindx_indices;
391:   PetscInt       *send_indices, rstart, *recv_indices, nrecvs, nsends;
392:   PetscInt       *tosizes, *fromsizes, i, j, *tosizes_tmp, *tooffsets_tmp, ito_ln;
393:   PetscMPIInt    *toranks, *fromranks, size, target_rank, *fromperm_newtoold, nto, nfrom;
394:   PetscLayout     isrmap;
395:   MPI_Comm        comm;
396:   PetscSF         sf;
397:   PetscSFNode    *iremote;

399:   PetscObjectGetComm((PetscObject)ito, &comm);
400:   MPI_Comm_size(comm, &size);
401:   ISGetLocalSize(ito, &ito_ln);
402:   ISGetLayout(ito, &isrmap);
403:   PetscLayoutGetRange(isrmap, &rstart, NULL);
404:   ISGetIndices(ito, &ito_indices);
405:   PetscCalloc2(size, &tosizes_tmp, size + 1, &tooffsets_tmp);
406:   for (i = 0; i < ito_ln; i++) {
407:     if (ito_indices[i] < 0) continue;
409:     tosizes_tmp[ito_indices[i]]++;
410:   }
411:   nto = 0;
412:   for (i = 0; i < size; i++) {
413:     tooffsets_tmp[i + 1] = tooffsets_tmp[i] + tosizes_tmp[i];
414:     if (tosizes_tmp[i] > 0) nto++;
415:   }
416:   PetscCalloc2(nto, &toranks, 2 * nto, &tosizes);
417:   nto = 0;
418:   for (i = 0; i < size; i++) {
419:     if (tosizes_tmp[i] > 0) {
420:       toranks[nto]         = i;
421:       tosizes[2 * nto]     = tosizes_tmp[i];   /* size */
422:       tosizes[2 * nto + 1] = tooffsets_tmp[i]; /* offset */
423:       nto++;
424:     }
425:   }
426:   nsends = tooffsets_tmp[size];
427:   PetscCalloc1(nsends, &send_indices);
428:   if (toindx) ISGetIndices(toindx, &toindx_indices);
429:   for (i = 0; i < ito_ln; i++) {
430:     if (ito_indices[i] < 0) continue;
431:     target_rank                              = ito_indices[i];
432:     send_indices[tooffsets_tmp[target_rank]] = toindx ? toindx_indices[i] : (i + rstart);
433:     tooffsets_tmp[target_rank]++;
434:   }
435:   if (toindx) ISRestoreIndices(toindx, &toindx_indices);
436:   ISRestoreIndices(ito, &ito_indices);
437:   PetscFree2(tosizes_tmp, tooffsets_tmp);
438:   PetscCommBuildTwoSided(comm, 2, MPIU_INT, nto, toranks, tosizes, &nfrom, &fromranks, &fromsizes);
439:   PetscFree2(toranks, tosizes);
440:   PetscMalloc1(nfrom, &fromperm_newtoold);
441:   for (i = 0; i < nfrom; i++) fromperm_newtoold[i] = i;
442:   PetscSortMPIIntWithArray(nfrom, fromranks, fromperm_newtoold);
443:   nrecvs = 0;
444:   for (i = 0; i < nfrom; i++) nrecvs += fromsizes[i * 2];
445:   PetscCalloc1(nrecvs, &recv_indices);
446:   PetscMalloc1(nrecvs, &iremote);
447:   nrecvs = 0;
448:   for (i = 0; i < nfrom; i++) {
449:     for (j = 0; j < fromsizes[2 * fromperm_newtoold[i]]; j++) {
450:       iremote[nrecvs].rank    = fromranks[i];
451:       iremote[nrecvs++].index = fromsizes[2 * fromperm_newtoold[i] + 1] + j;
452:     }
453:   }
454:   PetscSFCreate(comm, &sf);
455:   PetscSFSetGraph(sf, nsends, nrecvs, NULL, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER);
456:   PetscSFSetType(sf, PETSCSFBASIC);
457:   /* how to put a prefix ? */
458:   PetscSFSetFromOptions(sf);
459:   PetscSFBcastBegin(sf, MPIU_INT, send_indices, recv_indices, MPI_REPLACE);
460:   PetscSFBcastEnd(sf, MPIU_INT, send_indices, recv_indices, MPI_REPLACE);
461:   PetscSFDestroy(&sf);
462:   PetscFree(fromranks);
463:   PetscFree(fromsizes);
464:   PetscFree(fromperm_newtoold);
465:   PetscFree(send_indices);
466:   if (rows) {
467:     PetscSortInt(nrecvs, recv_indices);
468:     ISCreateGeneral(comm, nrecvs, recv_indices, PETSC_OWN_POINTER, rows);
469:   } else {
470:     PetscFree(recv_indices);
471:   }
472:   return 0;
473: }

475: /*@
476:     ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
477:     generates an IS that contains a new global node number for each index based
478:     on the partitioing.

480:     Collective on IS

482:     Input Parameters:
483: .   partitioning - a partitioning as generated by MatPartitioningApply()
484:                    or MatPartitioningApplyND()

486:     Output Parameter:
487: .   is - on each processor the index set that defines the global numbers
488:          (in the new numbering) for all the nodes currently (before the partitioning)
489:          on that processor

491:    Level: advanced

493: .seealso: `MatPartitioningCreate()`, `AOCreateBasic()`, `ISPartitioningCount()`

495: @*/
496: PetscErrorCode ISPartitioningToNumbering(IS part, IS *is)
497: {
498:   MPI_Comm        comm;
499:   IS              ndorder;
500:   PetscInt        i, np, npt, n, *starts = NULL, *sums = NULL, *lsizes = NULL, *newi = NULL;
501:   const PetscInt *indices = NULL;

505:   /* see if the partitioning comes from nested dissection */
506:   PetscObjectQuery((PetscObject)part, "_petsc_matpartitioning_ndorder", (PetscObject *)&ndorder);
507:   if (ndorder) {
508:     PetscObjectReference((PetscObject)ndorder);
509:     *is = ndorder;
510:     return 0;
511:   }

513:   PetscObjectGetComm((PetscObject)part, &comm);
514:   /* count the number of partitions, i.e., virtual processors */
515:   ISGetLocalSize(part, &n);
516:   ISGetIndices(part, &indices);
517:   np = 0;
518:   for (i = 0; i < n; i++) np = PetscMax(np, indices[i]);
519:   MPIU_Allreduce(&np, &npt, 1, MPIU_INT, MPI_MAX, comm);
520:   np = npt + 1; /* so that it looks like a MPI_Comm_size output */

522:   /*
523:         lsizes - number of elements of each partition on this particular processor
524:         sums - total number of "previous" nodes for any particular partition
525:         starts - global number of first element in each partition on this processor
526:   */
527:   PetscMalloc3(np, &lsizes, np, &starts, np, &sums);
528:   PetscArrayzero(lsizes, np);
529:   for (i = 0; i < n; i++) lsizes[indices[i]]++;
530:   MPIU_Allreduce(lsizes, sums, np, MPIU_INT, MPI_SUM, comm);
531:   MPI_Scan(lsizes, starts, np, MPIU_INT, MPI_SUM, comm);
532:   for (i = 0; i < np; i++) starts[i] -= lsizes[i];
533:   for (i = 1; i < np; i++) {
534:     sums[i] += sums[i - 1];
535:     starts[i] += sums[i - 1];
536:   }

538:   /*
539:       For each local index give it the new global number
540:   */
541:   PetscMalloc1(n, &newi);
542:   for (i = 0; i < n; i++) newi[i] = starts[indices[i]]++;
543:   PetscFree3(lsizes, starts, sums);

545:   ISRestoreIndices(part, &indices);
546:   ISCreateGeneral(comm, n, newi, PETSC_OWN_POINTER, is);
547:   ISSetPermutation(*is);
548:   return 0;
549: }

551: /*@
552:     ISPartitioningCount - Takes a ISPartitioning and determines the number of
553:     resulting elements on each (partition) process

555:     Collective on IS

557:     Input Parameters:
558: +   partitioning - a partitioning as generated by MatPartitioningApply() or
559:                    MatPartitioningApplyND()
560: -   len - length of the array count, this is the total number of partitions

562:     Output Parameter:
563: .   count - array of length size, to contain the number of elements assigned
564:         to each partition, where size is the number of partitions generated
565:          (see notes below).

567:    Level: advanced

569:     Notes:
570:         By default the number of partitions generated (and thus the length
571:         of count) is the size of the communicator associated with IS,
572:         but it can be set by MatPartitioningSetNParts. The resulting array
573:         of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.
574:         If the partitioning has been obtained by MatPartitioningApplyND(),
575:         the returned count does not include the separators.

577: .seealso: `MatPartitioningCreate()`, `AOCreateBasic()`, `ISPartitioningToNumbering()`,
578:           `MatPartitioningSetNParts()`, `MatPartitioningApply()`, `MatPartitioningApplyND()`

580: @*/
581: PetscErrorCode ISPartitioningCount(IS part, PetscInt len, PetscInt count[])
582: {
583:   MPI_Comm        comm;
584:   PetscInt        i, n, *lsizes;
585:   const PetscInt *indices;
586:   PetscMPIInt     npp;

588:   PetscObjectGetComm((PetscObject)part, &comm);
589:   if (len == PETSC_DEFAULT) {
590:     PetscMPIInt size;
591:     MPI_Comm_size(comm, &size);
592:     len = (PetscInt)size;
593:   }

595:   /* count the number of partitions */
596:   ISGetLocalSize(part, &n);
597:   ISGetIndices(part, &indices);
598:   if (PetscDefined(USE_DEBUG)) {
599:     PetscInt np = 0, npt;
600:     for (i = 0; i < n; i++) np = PetscMax(np, indices[i]);
601:     MPIU_Allreduce(&np, &npt, 1, MPIU_INT, MPI_MAX, comm);
602:     np = npt + 1; /* so that it looks like a MPI_Comm_size output */
604:   }

606:   /*
607:         lsizes - number of elements of each partition on this particular processor
608:         sums - total number of "previous" nodes for any particular partition
609:         starts - global number of first element in each partition on this processor
610:   */
611:   PetscCalloc1(len, &lsizes);
612:   for (i = 0; i < n; i++) {
613:     if (indices[i] > -1) lsizes[indices[i]]++;
614:   }
615:   ISRestoreIndices(part, &indices);
616:   PetscMPIIntCast(len, &npp);
617:   MPIU_Allreduce(lsizes, count, npp, MPIU_INT, MPI_SUM, comm);
618:   PetscFree(lsizes);
619:   return 0;
620: }

622: /*@
623:     ISAllGather - Given an index set (IS) on each processor, generates a large
624:     index set (same on each processor) by concatenating together each
625:     processors index set.

627:     Collective on IS

629:     Input Parameter:
630: .   is - the distributed index set

632:     Output Parameter:
633: .   isout - the concatenated index set (same on all processors)

635:     Notes:
636:     ISAllGather() is clearly not scalable for large index sets.

638:     The IS created on each processor must be created with a common
639:     communicator (e.g., PETSC_COMM_WORLD). If the index sets were created
640:     with PETSC_COMM_SELF, this routine will not work as expected, since
641:     each process will generate its own new IS that consists only of
642:     itself.

644:     The communicator for this new IS is PETSC_COMM_SELF

646:     Level: intermediate

648: .seealso: `ISCreateGeneral()`, `ISCreateStride()`, `ISCreateBlock()`
649: @*/
650: PetscErrorCode ISAllGather(IS is, IS *isout)
651: {
652:   PetscInt       *indices, n, i, N, step, first;
653:   const PetscInt *lindices;
654:   MPI_Comm        comm;
655:   PetscMPIInt     size, *sizes = NULL, *offsets = NULL, nn;
656:   PetscBool       stride;


661:   PetscObjectGetComm((PetscObject)is, &comm);
662:   MPI_Comm_size(comm, &size);
663:   ISGetLocalSize(is, &n);
664:   PetscObjectTypeCompare((PetscObject)is, ISSTRIDE, &stride);
665:   if (size == 1 && stride) { /* should handle parallel ISStride also */
666:     ISStrideGetInfo(is, &first, &step);
667:     ISCreateStride(PETSC_COMM_SELF, n, first, step, isout);
668:   } else {
669:     PetscMalloc2(size, &sizes, size, &offsets);

671:     PetscMPIIntCast(n, &nn);
672:     MPI_Allgather(&nn, 1, MPI_INT, sizes, 1, MPI_INT, comm);
673:     offsets[0] = 0;
674:     for (i = 1; i < size; i++) {
675:       PetscInt s = offsets[i - 1] + sizes[i - 1];
676:       PetscMPIIntCast(s, &offsets[i]);
677:     }
678:     N = offsets[size - 1] + sizes[size - 1];

680:     PetscMalloc1(N, &indices);
681:     ISGetIndices(is, &lindices);
682:     MPI_Allgatherv((void *)lindices, nn, MPIU_INT, indices, sizes, offsets, MPIU_INT, comm);
683:     ISRestoreIndices(is, &lindices);
684:     PetscFree2(sizes, offsets);

686:     ISCreateGeneral(PETSC_COMM_SELF, N, indices, PETSC_OWN_POINTER, isout);
687:   }
688:   return 0;
689: }

691: /*@C
692:     ISAllGatherColors - Given a a set of colors on each processor, generates a large
693:     set (same on each processor) by concatenating together each processors colors

695:     Collective

697:     Input Parameters:
698: +   comm - communicator to share the indices
699: .   n - local size of set
700: -   lindices - local colors

702:     Output Parameters:
703: +   outN - total number of indices
704: -   outindices - all of the colors

706:     Notes:
707:     ISAllGatherColors() is clearly not scalable for large index sets.

709:     Level: intermediate

711: .seealso: `ISCreateGeneral()`, `ISCreateStride()`, `ISCreateBlock()`, `ISAllGather()`
712: @*/
713: PetscErrorCode ISAllGatherColors(MPI_Comm comm, PetscInt n, ISColoringValue *lindices, PetscInt *outN, ISColoringValue *outindices[])
714: {
715:   ISColoringValue *indices;
716:   PetscInt         i, N;
717:   PetscMPIInt      size, *offsets = NULL, *sizes = NULL, nn = n;

719:   MPI_Comm_size(comm, &size);
720:   PetscMalloc2(size, &sizes, size, &offsets);

722:   MPI_Allgather(&nn, 1, MPI_INT, sizes, 1, MPI_INT, comm);
723:   offsets[0] = 0;
724:   for (i = 1; i < size; i++) offsets[i] = offsets[i - 1] + sizes[i - 1];
725:   N = offsets[size - 1] + sizes[size - 1];
726:   PetscFree2(sizes, offsets);

728:   PetscMalloc1(N + 1, &indices);
729:   MPI_Allgatherv(lindices, (PetscMPIInt)n, MPIU_COLORING_VALUE, indices, sizes, offsets, MPIU_COLORING_VALUE, comm);

731:   *outindices = indices;
732:   if (outN) *outN = N;
733:   return 0;
734: }

736: /*@
737:     ISComplement - Given an index set (IS) generates the complement index set. That is all
738:        all indices that are NOT in the given set.

740:     Collective on IS

742:     Input Parameters:
743: +   is - the index set
744: .   nmin - the first index desired in the local part of the complement
745: -   nmax - the largest index desired in the local part of the complement (note that all indices in is must be greater or equal to nmin and less than nmax)

747:     Output Parameter:
748: .   isout - the complement

750:     Notes:
751:     The communicator for this new IS is the same as for the input IS

753:       For a parallel IS, this will generate the local part of the complement on each process

755:       To generate the entire complement (on each process) of a parallel IS, first call ISAllGather() and then
756:     call this routine.

758:     Level: intermediate

760: .seealso: `ISCreateGeneral()`, `ISCreateStride()`, `ISCreateBlock()`, `ISAllGather()`
761: @*/
762: PetscErrorCode ISComplement(IS is, PetscInt nmin, PetscInt nmax, IS *isout)
763: {
764:   const PetscInt *indices;
765:   PetscInt        n, i, j, unique, cnt, *nindices;
766:   PetscBool       sorted;

772:   ISSorted(is, &sorted);

775:   ISGetLocalSize(is, &n);
776:   ISGetIndices(is, &indices);
777:   if (PetscDefined(USE_DEBUG)) {
778:     for (i = 0; i < n; i++) {
781:     }
782:   }
783:   /* Count number of unique entries */
784:   unique = (n > 0);
785:   for (i = 0; i < n - 1; i++) {
786:     if (indices[i + 1] != indices[i]) unique++;
787:   }
788:   PetscMalloc1(nmax - nmin - unique, &nindices);
789:   cnt = 0;
790:   for (i = nmin, j = 0; i < nmax; i++) {
791:     if (j < n && i == indices[j]) do {
792:         j++;
793:       } while (j < n && i == indices[j]);
794:     else nindices[cnt++] = i;
795:   }
797:   ISCreateGeneral(PetscObjectComm((PetscObject)is), cnt, nindices, PETSC_OWN_POINTER, isout);
798:   ISRestoreIndices(is, &indices);
799:   return 0;
800: }