Actual source code: isltog.c

  1: /*$Id: isltog.c,v 1.64 2001/03/23 23:21:16 balay Exp $*/

 3:  #include petscsys.h
 4:  #include src/vec/is/isimpl.h

  6: /*@C
  7:     ISLocalToGlobalMappingGetSize - Gets the local size of a local to global mapping.

  9:     Not Collective

 11:     Input Parameter:
 12: .   ltog - local to global mapping

 14:     Output Parameter:
 15: .   n - the number of entries in the local mapping

 17:     Level: advanced

 19:     Concepts: mapping^local to global

 21: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
 22: @*/
 23: int ISLocalToGlobalMappingGetSize(ISLocalToGlobalMapping mapping,int *n)
 24: {
 27:   *n = mapping->n;
 28:   return(0);
 29: }

 31: /*@C
 32:     ISLocalToGlobalMappingView - View a local to global mapping

 34:     Not Collective

 36:     Input Parameters:
 37: +   ltog - local to global mapping
 38: -   viewer - viewer

 40:     Level: advanced

 42:     Concepts: mapping^local to global

 44: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
 45: @*/
 46: int ISLocalToGlobalMappingView(ISLocalToGlobalMapping mapping,PetscViewer viewer)
 47: {
 48:   int        i,ierr,rank;
 49:   PetscTruth isascii;

 53:   if (!viewer) viewer = PETSC_VIEWER_STDOUT_(mapping->comm);

 57:   MPI_Comm_rank(mapping->comm,&rank);
 58:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
 59:   if (isascii) {
 60:     for (i=0; i<mapping->n; i++) {
 61:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %d %dn",rank,i,mapping->indices[i]);
 62:     }
 63:     PetscViewerFlush(viewer);
 64:   } else {
 65:     SETERRQ1(1,"Viewer type %s not supported for ISLocalToGlobalMapping",((PetscObject)viewer)->type_name);
 66:   }

 68:   return(0);
 69: }

 71: /*@C
 72:     ISLocalToGlobalMappingCreateIS - Creates a mapping between a local (0 to n)
 73:     ordering and a global parallel ordering.

 75:     Not collective

 77:     Input Parameter:
 78: .   is - index set containing the global numbers for each local

 80:     Output Parameter:
 81: .   mapping - new mapping data structure

 83:     Level: advanced

 85:     Concepts: mapping^local to global

 87: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
 88: @*/
 89: int ISLocalToGlobalMappingCreateIS(IS is,ISLocalToGlobalMapping *mapping)
 90: {
 91:   int      n,*indices,ierr;
 92:   MPI_Comm comm;


 97:   PetscObjectGetComm((PetscObject)is,&comm);
 98:   ISGetLocalSize(is,&n);
 99:   ISGetIndices(is,&indices);
100:   ISLocalToGlobalMappingCreate(comm,n,indices,mapping);
101:   ISRestoreIndices(is,&indices);

103:   return(0);
104: }

106: /*@C
107:     ISLocalToGlobalMappingCreate - Creates a mapping between a local (0 to n)
108:     ordering and a global parallel ordering.

110:     Not Collective, but communicator may have more than one process

112:     Input Parameters:
113: +   comm - MPI communicator
114: .   n - the number of local elements
115: -   indices - the global index for each local element

117:     Output Parameter:
118: .   mapping - new mapping data structure

120:     Level: advanced

122:     Concepts: mapping^local to global

124: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS()
125: @*/
126: int ISLocalToGlobalMappingCreate(MPI_Comm cm,int n,const int indices[],ISLocalToGlobalMapping *mapping)
127: {


134:   PetscHeaderCreate(*mapping,_p_ISLocalToGlobalMapping,int,IS_LTOGM_COOKIE,0,"ISLocalToGlobalMapping",
135:                     cm,ISLocalToGlobalMappingDestroy,ISLocalToGlobalMappingView);
136:   PetscLogObjectCreate(*mapping);
137:   PetscLogObjectMemory(*mapping,sizeof(struct _p_ISLocalToGlobalMapping)+n*sizeof(int));

139:   (*mapping)->n       = n;
140:   PetscMalloc((n+1)*sizeof(int),&(*mapping)->indices);
141:   PetscMemcpy((*mapping)->indices,indices,n*sizeof(int));

143:   /*
144:       Do not create the global to local mapping. This is only created if 
145:      ISGlobalToLocalMapping() is called 
146:   */
147:   (*mapping)->globals = 0;
148:   return(0);
149: }

151: /*@C
152:     ISLocalToGlobalMappingBlock - Creates a blocked index version of an 
153:        ISLocalToGlobalMapping that is appropriate for MatSetLocalToGlobalMappingBlock()
154:        and VecSetLocalToGlobalMappingBlock().

156:     Not Collective, but communicator may have more than one process

158:     Input Parameters:
159: +    inmap - original point-wise mapping
160: -    bs - block size

162:     Output Parameter:
163: .   outmap - block based mapping

165:     Level: advanced

167:     Concepts: mapping^local to global

169: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingCreateIS()
170: @*/
171: int ISLocalToGlobalMappingBlock(ISLocalToGlobalMapping inmap,int bs,ISLocalToGlobalMapping *outmap)
172: {
173:   int ierr,*ii,i,n;


177:   if (bs > 1) {
178:     n    = inmap->n/bs;
179:     PetscMalloc(n*sizeof(int),&ii);
180:     for (i=0; i<n; i++) {
181:       ii[i] = inmap->indices[bs*i]/bs;
182:     }
183:     ISLocalToGlobalMappingCreate(inmap->comm,n,ii,outmap);
184:     PetscFree(ii);
185:   } else {
186:     *outmap = inmap;
187:     ierr    = PetscObjectReference((PetscObject)inmap);
188:   }
189:   return(0);
190: }
191: 
192: /*@
193:    ISLocalToGlobalMappingDestroy - Destroys a mapping between a local (0 to n)
194:    ordering and a global parallel ordering.

196:    Note Collective

198:    Input Parameters:
199: .  mapping - mapping data structure

201:    Level: advanced

203: .seealso: ISLocalToGlobalMappingCreate()
204: @*/
205: int ISLocalToGlobalMappingDestroy(ISLocalToGlobalMapping mapping)
206: {
210:   if (--mapping->refct > 0) return(0);
211:   if (mapping->refct < 0) {
212:     SETERRQ(1,"Mapping already destroyed");
213:   }

215:   PetscFree(mapping->indices);
216:   if (mapping->globals) {PetscFree(mapping->globals);}
217:   PetscLogObjectDestroy(mapping);
218:   PetscHeaderDestroy(mapping);
219:   return(0);
220: }
221: 
222: /*@
223:     ISLocalToGlobalMappingApplyIS - Creates from an IS in the local numbering
224:     a new index set using the global numbering defined in an ISLocalToGlobalMapping
225:     context.

227:     Not collective

229:     Input Parameters:
230: +   mapping - mapping between local and global numbering
231: -   is - index set in local numbering

233:     Output Parameters:
234: .   newis - index set in global numbering

236:     Level: advanced

238:     Concepts: mapping^local to global

240: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
241:           ISLocalToGlobalMappingDestroy(), ISGlobalToLocalMappingApply()
242: @*/
243: int ISLocalToGlobalMappingApplyIS(ISLocalToGlobalMapping mapping,IS is,IS *newis)
244: {
245:   int ierr,n,i,*idxin,*idxmap,*idxout,Nmax = mapping->n;


252:   ierr   = ISGetLocalSize(is,&n);
253:   ierr   = ISGetIndices(is,&idxin);
254:   idxmap = mapping->indices;
255: 
256:   PetscMalloc((n+1)*sizeof(int),&idxout);
257:   for (i=0; i<n; i++) {
258:     if (idxin[i] >= Nmax) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Local index %d too large %d (max) at %d",idxin[i],Nmax,i);
259:     idxout[i] = idxmap[idxin[i]];
260:   }
261:   ISRestoreIndices(is,&idxin);
262:   ISCreateGeneral(PETSC_COMM_SELF,n,idxout,newis);
263:   PetscFree(idxout);
264:   return(0);
265: }

267: /*MC
268:    ISLocalToGlobalMappingApply - Takes a list of integers in a local numbering
269:    and converts them to the global numbering.

271:    Not collective

273:    Input Parameters:
274: +  mapping - the local to global mapping context
275: .  N - number of integers
276: -  in - input indices in local numbering

278:    Output Parameter:
279: .  out - indices in global numbering

281:    Synopsis:
282:    ISLocalToGlobalMappingApply(ISLocalToGlobalMapping mapping,int N,int in[],int out[])

284:    Notes: 
285:    The in and out array parameters may be identical.

287:    Level: advanced

289: .seealso: ISLocalToGlobalMappingCreate(),ISLocalToGlobalMappingDestroy(), 
290:           ISLocalToGlobalMappingApplyIS(),AOCreateBasic(),AOApplicationToPetsc(),
291:           AOPetscToApplication(), ISGlobalToLocalMappingApply()

293:     Concepts: mapping^local to global

295: M*/

297: /* -----------------------------------------------------------------------------------------*/

299: /*
300:     Creates the global fields in the ISLocalToGlobalMapping structure
301: */
302: static int ISGlobalToLocalMappingSetUp_Private(ISLocalToGlobalMapping mapping)
303: {
304:   int ierr,i,*idx = mapping->indices,n = mapping->n,end,start,*globals;

307:   end   = 0;
308:   start = 100000000;

310:   for (i=0; i<n; i++) {
311:     if (idx[i] < 0) continue;
312:     if (idx[i] < start) start = idx[i];
313:     if (idx[i] > end)   end   = idx[i];
314:   }
315:   if (start > end) {start = 0; end = -1;}
316:   mapping->globalstart = start;
317:   mapping->globalend   = end;

319:   ierr             = PetscMalloc((end-start+2)*sizeof(int),&globals);
320:   mapping->globals = globals;
321:   for (i=0; i<end-start+1; i++) {
322:     globals[i] = -1;
323:   }
324:   for (i=0; i<n; i++) {
325:     if (idx[i] < 0) continue;
326:     globals[idx[i] - start] = i;
327:   }

329:   PetscLogObjectMemory(mapping,(end-start+1)*sizeof(int));
330:   return(0);
331: }

333: /*@
334:     ISGlobalToLocalMappingApply - Provides the local numbering for a list of integers
335:     specified with a global numbering.

337:     Not collective

339:     Input Parameters:
340: +   mapping - mapping between local and global numbering
341: .   type - IS_GTOLM_MASK - replaces global indices with no local value with -1
342:            IS_GTOLM_DROP - drops the indices with no local value from the output list
343: .   n - number of global indices to map
344: -   idx - global indices to map

346:     Output Parameters:
347: +   nout - number of indices in output array (if type == IS_GTOLM_MASK then nout = n)
348: -   idxout - local index of each global index, one must pass in an array long enough 
349:              to hold all the indices. You can call ISGlobalToLocalMappingApply() with 
350:              idxout == PETSC_NULL to determine the required length (returned in nout)
351:              and then allocate the required space and call ISGlobalToLocalMappingApply()
352:              a second time to set the values.

354:     Notes:
355:     Either nout or idxout may be PETSC_NULL. idx and idxout may be identical.

357:     This is not scalable in memory usage. Each processor requires O(Nglobal) size 
358:     array to compute these.

360:     Level: advanced

362:     Concepts: mapping^global to local

364: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
365:           ISLocalToGlobalMappingDestroy()
366: @*/
367: int ISGlobalToLocalMappingApply(ISLocalToGlobalMapping mapping,ISGlobalToLocalMappingType type,
368:                                   int n,const int idx[],int *nout,int idxout[])
369: {
370:   int i,ierr,*globals,nf = 0,tmp,start,end;

373:   if (!mapping->globals) {
374:     ISGlobalToLocalMappingSetUp_Private(mapping);
375:   }
376:   globals = mapping->globals;
377:   start   = mapping->globalstart;
378:   end     = mapping->globalend;

380:   if (type == IS_GTOLM_MASK) {
381:     if (idxout) {
382:       for (i=0; i<n; i++) {
383:         if (idx[i] < 0) idxout[i] = idx[i];
384:         else if (idx[i] < start) idxout[i] = -1;
385:         else if (idx[i] > end)   idxout[i] = -1;
386:         else                     idxout[i] = globals[idx[i] - start];
387:       }
388:     }
389:     if (nout) *nout = n;
390:   } else {
391:     if (idxout) {
392:       for (i=0; i<n; i++) {
393:         if (idx[i] < 0) continue;
394:         if (idx[i] < start) continue;
395:         if (idx[i] > end) continue;
396:         tmp = globals[idx[i] - start];
397:         if (tmp < 0) continue;
398:         idxout[nf++] = tmp;
399:       }
400:     } else {
401:       for (i=0; i<n; i++) {
402:         if (idx[i] < 0) continue;
403:         if (idx[i] < start) continue;
404:         if (idx[i] > end) continue;
405:         tmp = globals[idx[i] - start];
406:         if (tmp < 0) continue;
407:         nf++;
408:       }
409:     }
410:     if (nout) *nout = nf;
411:   }

413:   return(0);
414: }

416: /*@C
417:     ISLocalToGlobalMappingGetInfo - Gets the neighbor information for each processor and 
418:      each index shared by more than one processor 

420:     Collective on ISLocalToGlobalMapping

422:     Input Parameters:
423: .   mapping - the mapping from local to global indexing

425:     Output Parameter:
426: +   nproc - number of processors that are connected to this one
427: .   proc - neighboring processors
428: .   numproc - number of indices for each subdomain (processor)
429: -   indices - indices of local nodes shared with neighbor (sorted by global numbering)

431:     Level: advanced

433:     Concepts: mapping^local to global

435: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
436:           ISLocalToGlobalMappingRestoreInfo()
437: @*/
438: int ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping mapping,int *nproc,int **procs,int **numprocs,int ***indices)
439: {
440:   int         i,n = mapping->n,ierr,Ng,ng,max = 0,*lindices = mapping->indices;
441:   int         size,rank,*nprocs,*owner,nsends,*sends,j,*starts,*work,nmax,nrecvs,*recvs,proc;
442:   int         tag1,tag2,tag3,cnt,*len,*source,imdex,scale,*ownedsenders,*nownedsenders,rstart,nowned;
443:   int         node,nownedm,nt,*sends2,nsends2,*starts2,*lens2,*dest,nrecvs2,*starts3,*recvs2,k,*bprocs,*tmp;
444:   int         first_procs,first_numprocs,*first_indices;
445:   MPI_Request *recv_waits,*send_waits;
446:   MPI_Status  recv_status,*send_status,*recv_statuses;
447:   MPI_Comm    comm = mapping->comm;
448:   PetscTruth  debug = PETSC_FALSE;

451:   ierr   = MPI_Comm_size(comm,&size);
452:   ierr   = MPI_Comm_rank(comm,&rank);
453:   if (size == 1) {
454:     *nproc         = 0;
455:     *procs         = PETSC_NULL;
456:     ierr           = PetscMalloc(sizeof(int),numprocs);
457:     (*numprocs)[0] = 0;
458:     ierr           = PetscMalloc(sizeof(int*),indices);
459:     (*indices)[0]  = PETSC_NULL;
460:     return(0);
461:   }

463:   PetscOptionsHasName(PETSC_NULL,"-islocaltoglobalmappinggetinfo_debug",&debug);

465:   /*
466:     Notes on ISLocalToGlobalMappingGetInfo

468:     globally owned node - the nodes that have been assigned to this processor in global
469:            numbering, just for this routine.

471:     nontrivial globally owned node - node assigned to this processor that is on a subdomain
472:            boundary (i.e. is has more than one local owner)

474:     locally owned node - node that exists on this processors subdomain

476:     nontrivial locally owned node - node that is not in the interior (i.e. has more than one
477:            local subdomain
478:   */
479:   PetscObjectGetNewTag((PetscObject)mapping,&tag1);
480:   PetscObjectGetNewTag((PetscObject)mapping,&tag2);
481:   PetscObjectGetNewTag((PetscObject)mapping,&tag3);

483:   for (i=0; i<n; i++) {
484:     if (lindices[i] > max) max = lindices[i];
485:   }
486:   ierr   = MPI_Allreduce(&max,&Ng,1,MPI_INT,MPI_MAX,comm);
487:   Ng++;
488:   ierr   = MPI_Comm_size(comm,&size);
489:   ierr   = MPI_Comm_rank(comm,&rank);
490:   scale  = Ng/size + 1;
491:   ng     = scale; if (rank == size-1) ng = Ng - scale*(size-1); ng = PetscMax(1,ng);
492:   rstart = scale*rank;

494:   /* determine ownership ranges of global indices */
495:   PetscMalloc((2*size+1)*sizeof(int),&nprocs);
496:   PetscMemzero(nprocs,2*size*sizeof(int));

498:   /* determine owners of each local node  */
499:   PetscMalloc((n+1)*sizeof(int),&owner);
500:   for (i=0; i<n; i++) {
501:     proc              = lindices[i]/scale; /* processor that globally owns this index */
502:     nprocs[size+proc] = 1;                 /* processor globally owns at least one of ours */
503:     owner[i]          = proc;
504:     nprocs[proc]++;                        /* count of how many that processor globally owns of ours */
505:   }
506:   nsends = 0; for (i=0; i<size; i++) nsends += nprocs[size + i];
507:   PetscLogInfo(0,"ISLocalToGlobalMappingGetInfo: Number of global owners for my local data %dn",nsends);

509:   /* inform other processors of number of messages and max length*/
510:   PetscMalloc(2*size*sizeof(int),&work);
511:   ierr   = MPI_Allreduce(nprocs,work,2*size,MPI_INT,PetscMaxSum_Op,comm);
512:   nmax   = work[rank];
513:   nrecvs = work[size+rank];
514:   ierr   = PetscFree(work);
515:   PetscLogInfo(0,"ISLocalToGlobalMappingGetInfo: Number of local owners for my global data %dn",nrecvs);

517:   /* post receives for owned rows */
518:   PetscMalloc((2*nrecvs+1)*(nmax+1)*sizeof(int),&recvs);
519:   PetscMalloc((nrecvs+1)*sizeof(MPI_Request),&recv_waits);
520:   for (i=0; i<nrecvs; i++) {
521:     MPI_Irecv(recvs+2*nmax*i,2*nmax,MPI_INT,MPI_ANY_SOURCE,tag1,comm,recv_waits+i);
522:   }

524:   /* pack messages containing lists of local nodes to owners */
525:   ierr       = PetscMalloc((2*n+1)*sizeof(int),&sends);
526:   ierr       = PetscMalloc((size+1)*sizeof(int),&starts);
527:   starts[0]  = 0;
528:   for (i=1; i<size; i++) { starts[i] = starts[i-1] + 2*nprocs[i-1];}
529:   for (i=0; i<n; i++) {
530:     sends[starts[owner[i]]++] = lindices[i];
531:     sends[starts[owner[i]]++] = i;
532:   }
533:   PetscFree(owner);
534:   starts[0]  = 0;
535:   for (i=1; i<size; i++) { starts[i] = starts[i-1] + 2*nprocs[i-1];}

537:   /* send the messages */
538:   PetscMalloc((nsends+1)*sizeof(MPI_Request),&send_waits);
539:   PetscMalloc((nsends+1)*sizeof(int),&dest);
540:   cnt = 0;
541:   for (i=0; i<size; i++) {
542:     if (nprocs[i]) {
543:       ierr      = MPI_Isend(sends+starts[i],2*nprocs[i],MPI_INT,i,tag1,comm,send_waits+cnt);
544:       dest[cnt] = i;
545:       cnt++;
546:     }
547:   }
548:   PetscFree(starts);

550:   /* wait on receives */
551:   PetscMalloc((2*nrecvs+1)*sizeof(int),&source);
552:   len  = source + nrecvs;
553:   cnt  = nrecvs;
554:   PetscMalloc((ng+1)*sizeof(int),&nownedsenders);
555:   PetscMemzero(nownedsenders,ng*sizeof(int));
556:   while (cnt) {
557:     MPI_Waitany(nrecvs,recv_waits,&imdex,&recv_status);
558:     /* unpack receives into our local space */
559:     ierr           = MPI_Get_count(&recv_status,MPI_INT,&len[imdex]);
560:     source[imdex]  = recv_status.MPI_SOURCE;
561:     len[imdex]     = len[imdex]/2;
562:     /* count how many local owners for each of my global owned indices */
563:     for (i=0; i<len[imdex]; i++) nownedsenders[recvs[2*imdex*nmax+2*i]-rstart]++;
564:     cnt--;
565:   }
566:   PetscFree(recv_waits);

568:   /* count how many globally owned indices are on an edge multiplied by how many processors own them. */
569:   nowned  = 0;
570:   nownedm = 0;
571:   for (i=0; i<ng; i++) {
572:     if (nownedsenders[i] > 1) {nownedm += nownedsenders[i]; nowned++;}
573:   }

575:   /* create single array to contain rank of all local owners of each globally owned index */
576:   ierr      = PetscMalloc((nownedm+1)*sizeof(int),&ownedsenders);
577:   ierr      = PetscMalloc((ng+1)*sizeof(int),&starts);
578:   starts[0] = 0;
579:   for (i=1; i<ng; i++) {
580:     if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
581:     else starts[i] = starts[i-1];
582:   }

584:   /* for each nontrival globally owned node list all arriving processors */
585:   for (i=0; i<nrecvs; i++) {
586:     for (j=0; j<len[i]; j++) {
587:       node = recvs[2*i*nmax+2*j]-rstart;
588:       if (nownedsenders[node] > 1) {
589:         ownedsenders[starts[node]++] = source[i];
590:       }
591:     }
592:   }

594:   if (debug) { /* -----------------------------------  */
595:     starts[0]    = 0;
596:     for (i=1; i<ng; i++) {
597:       if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
598:       else starts[i] = starts[i-1];
599:     }
600:     for (i=0; i<ng; i++) {
601:       if (nownedsenders[i] > 1) {
602:         PetscSynchronizedPrintf(comm,"[%d] global node %d local owner processors: ",rank,i+rstart);
603:         for (j=0; j<nownedsenders[i]; j++) {
604:           PetscSynchronizedPrintf(comm,"%d ",ownedsenders[starts[i]+j]);
605:         }
606:         PetscSynchronizedPrintf(comm,"n");
607:       }
608:     }
609:     PetscSynchronizedFlush(comm);
610:   }/* -----------------------------------  */

612:   /* wait on original sends */
613:   if (nsends) {
614:     PetscMalloc(nsends*sizeof(MPI_Status),&send_status);
615:     MPI_Waitall(nsends,send_waits,send_status);
616:     PetscFree(send_status);
617:   }
618:   PetscFree(send_waits);
619:   PetscFree(sends);
620:   PetscFree(nprocs);

622:   /* pack messages to send back to local owners */
623:   starts[0]    = 0;
624:   for (i=1; i<ng; i++) {
625:     if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
626:     else starts[i] = starts[i-1];
627:   }
628:   nsends2 = nrecvs;
629:   ierr    = PetscMalloc((nsends2+1)*sizeof(int),&nprocs); /* length of each message */
630:   for (i=0; i<nrecvs; i++) {
631:     nprocs[i] = 1;
632:     for (j=0; j<len[i]; j++) {
633:       node = recvs[2*i*nmax+2*j]-rstart;
634:       if (nownedsenders[node] > 1) {
635:         nprocs[i] += 2 + nownedsenders[node];
636:       }
637:     }
638:   }
639:   nt = 0; for (i=0; i<nsends2; i++) nt += nprocs[i];
640:   PetscMalloc((nt+1)*sizeof(int),&sends2);
641:   PetscMalloc((nsends2+1)*sizeof(int),&starts2);
642:   starts2[0] = 0; for (i=1; i<nsends2; i++) starts2[i] = starts2[i-1] + nprocs[i-1];
643:   /*
644:      Each message is 1 + nprocs[i] long, and consists of 
645:        (0) the number of nodes being sent back 
646:        (1) the local node number,
647:        (2) the number of processors sharing it,
648:        (3) the processors sharing it
649:   */
650:   for (i=0; i<nsends2; i++) {
651:     cnt = 1;
652:     sends2[starts2[i]] = 0;
653:     for (j=0; j<len[i]; j++) {
654:       node = recvs[2*i*nmax+2*j]-rstart;
655:       if (nownedsenders[node] > 1) {
656:         sends2[starts2[i]]++;
657:         sends2[starts2[i]+cnt++] = recvs[2*i*nmax+2*j+1];
658:         sends2[starts2[i]+cnt++] = nownedsenders[node];
659:         PetscMemcpy(&sends2[starts2[i]+cnt],&ownedsenders[starts[node]],nownedsenders[node]*sizeof(int));
660:         cnt += nownedsenders[node];
661:       }
662:     }
663:   }

665:   /* send the message lengths */
666:   for (i=0; i<nsends2; i++) {
667:     MPI_Send(&nprocs[i],1,MPI_INT,source[i],tag2,comm);
668:   }

670:   /* receive the message lengths */
671:   nrecvs2 = nsends;
672:   PetscMalloc((nrecvs2+1)*sizeof(int),&lens2);
673:   PetscMalloc((nrecvs2+1)*sizeof(int),&starts3);
674:   nt      = 0;
675:   for (i=0; i<nrecvs2; i++) {
676:      MPI_Recv(&lens2[i],1,MPI_INT,dest[i],tag2,comm,&recv_status);
677:     nt   += lens2[i];
678:   }
679:   starts3[0] = 0;
680:   for (i=0; i<nrecvs2-1; i++) {
681:     starts3[i+1] = starts3[i] + lens2[i];
682:   }
683:   PetscMalloc((nt+1)*sizeof(int),&recvs2);
684:   PetscMalloc((nrecvs2+1)*sizeof(MPI_Request),&recv_waits);
685:   for (i=0; i<nrecvs2; i++) {
686:     MPI_Irecv(recvs2+starts3[i],lens2[i],MPI_INT,dest[i],tag3,comm,recv_waits+i);
687:   }
688: 
689:   /* send the messages */
690:   PetscMalloc((nsends2+1)*sizeof(MPI_Request),&send_waits);
691:   for (i=0; i<nsends2; i++) {
692:     MPI_Isend(sends2+starts2[i],nprocs[i],MPI_INT,source[i],tag3,comm,send_waits+i);
693:   }

695:   /* wait on receives */
696:   PetscMalloc((nrecvs2+1)*sizeof(MPI_Status),&recv_statuses);
697:   MPI_Waitall(nrecvs2,recv_waits,recv_statuses);
698:   PetscFree(recv_statuses);
699:   PetscFree(recv_waits);
700:   PetscFree(nprocs);

702:   if (debug) { /* -----------------------------------  */
703:     cnt = 0;
704:     for (i=0; i<nrecvs2; i++) {
705:       nt = recvs2[cnt++];
706:       for (j=0; j<nt; j++) {
707:         PetscSynchronizedPrintf(comm,"[%d] local node %d number of subdomains %d: ",rank,recvs2[cnt],recvs2[cnt+1]);
708:         for (k=0; k<recvs2[cnt+1]; k++) {
709:           PetscSynchronizedPrintf(comm,"%d ",recvs2[cnt+2+k]);
710:         }
711:         cnt += 2 + recvs2[cnt+1];
712:         PetscSynchronizedPrintf(comm,"n");
713:       }
714:     }
715:     PetscSynchronizedFlush(comm);
716:   } /* -----------------------------------  */

718:   /* count number subdomains for each local node */
719:   PetscMalloc(size*sizeof(int),&nprocs);
720:   PetscMemzero(nprocs,size*sizeof(int));
721:   cnt  = 0;
722:   for (i=0; i<nrecvs2; i++) {
723:     nt = recvs2[cnt++];
724:     for (j=0; j<nt; j++) {
725:       for (k=0; k<recvs2[cnt+1]; k++) {
726:         nprocs[recvs2[cnt+2+k]]++;
727:       }
728:       cnt += 2 + recvs2[cnt+1];
729:     }
730:   }
731:   nt = 0; for (i=0; i<size; i++) nt += (nprocs[i] > 0);
732:   *nproc    = nt;
733:   PetscMalloc((nt+1)*sizeof(int),procs);
734:   PetscMalloc((nt+1)*sizeof(int),numprocs);
735:   PetscMalloc((nt+1)*sizeof(int*),indices);
736:   PetscMalloc(size*sizeof(int),&bprocs);
737:   cnt       = 0;
738:   for (i=0; i<size; i++) {
739:     if (nprocs[i] > 0) {
740:       bprocs[i]        = cnt;
741:       (*procs)[cnt]    = i;
742:       (*numprocs)[cnt] = nprocs[i];
743:       ierr             = PetscMalloc(nprocs[i]*sizeof(int),&(*indices)[cnt]);
744:       cnt++;
745:     }
746:   }

748:   /* make the list of subdomains for each nontrivial local node */
749:   PetscMemzero(*numprocs,nt*sizeof(int));
750:   cnt  = 0;
751:   for (i=0; i<nrecvs2; i++) {
752:     nt = recvs2[cnt++];
753:     for (j=0; j<nt; j++) {
754:       for (k=0; k<recvs2[cnt+1]; k++) {
755:         (*indices)[bprocs[recvs2[cnt+2+k]]][(*numprocs)[bprocs[recvs2[cnt+2+k]]]++] = recvs2[cnt];
756:       }
757:       cnt += 2 + recvs2[cnt+1];
758:     }
759:   }
760:   PetscFree(bprocs);
761:   PetscFree(recvs2);

763:   /* sort the node indexing by their global numbers */
764:   nt = *nproc;
765:   for (i=0; i<nt; i++) {
766:     PetscMalloc(((*numprocs)[i])*sizeof(int),&tmp);
767:     for (j=0; j<(*numprocs)[i]; j++) {
768:       tmp[j] = lindices[(*indices)[i][j]];
769:     }
770:     PetscSortIntWithArray((*numprocs)[i],tmp,(*indices)[i]);
771:     PetscFree(tmp);
772:   }

774:   if (debug) { /* -----------------------------------  */
775:     nt = *nproc;
776:     for (i=0; i<nt; i++) {
777:       PetscSynchronizedPrintf(comm,"[%d] subdomain %d number of indices %d: ",rank,(*procs)[i],(*numprocs)[i]);
778:       for (j=0; j<(*numprocs)[i]; j++) {
779:         PetscSynchronizedPrintf(comm,"%d ",(*indices)[i][j]);
780:       }
781:       PetscSynchronizedPrintf(comm,"n");
782:     }
783:     PetscSynchronizedFlush(comm);
784:   } /* -----------------------------------  */

786:   /* wait on sends */
787:   if (nsends2) {
788:     PetscMalloc(nsends2*sizeof(MPI_Status),&send_status);
789:     MPI_Waitall(nsends2,send_waits,send_status);
790:     PetscFree(send_status);
791:   }

793:   PetscFree(starts3);
794:   PetscFree(dest);
795:   PetscFree(send_waits);

797:   PetscFree(nownedsenders);
798:   PetscFree(ownedsenders);
799:   PetscFree(starts);
800:   PetscFree(starts2);
801:   PetscFree(lens2);

803:   PetscFree(source);
804:   PetscFree(recvs);
805:   PetscFree(nprocs);
806:   PetscFree(sends2);

808:   /* put the information about myself as the first entry in the list */
809:   first_procs    = (*procs)[0];
810:   first_numprocs = (*numprocs)[0];
811:   first_indices  = (*indices)[0];
812:   for (i=0; i<*nproc; i++) {
813:     if ((*procs)[i] == rank) {
814:       (*procs)[0]    = (*procs)[i];
815:       (*numprocs)[0] = (*numprocs)[i];
816:       (*indices)[0]  = (*indices)[i];
817:       (*procs)[i]    = first_procs;
818:       (*numprocs)[i] = first_numprocs;
819:       (*indices)[i]  = first_indices;
820:       break;
821:     }
822:   }

824:   return(0);
825: }

827: /*@C
828:     ISLocalToGlobalMappingRestoreInfo - Frees the memory allocated by ISLocalToGlobalMappingGetInfo()

830:     Collective on ISLocalToGlobalMapping

832:     Input Parameters:
833: .   mapping - the mapping from local to global indexing

835:     Output Parameter:
836: +   nproc - number of processors that are connected to this one
837: .   proc - neighboring processors
838: .   numproc - number of indices for each processor
839: -   indices - indices of local nodes shared with neighbor (sorted by global numbering)

841:     Level: advanced

843: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
844:           ISLocalToGlobalMappingGetInfo()
845: @*/
846: int ISLocalToGlobalMappingRestoreInfo(ISLocalToGlobalMapping mapping,int *nproc,int **procs,int **numprocs,int ***indices)
847: {
848:   int ierr,i;

851:   if (*procs) {PetscFree(*procs);}
852:   if (*numprocs) {PetscFree(*numprocs);}
853:   if (*indices) {
854:     if ((*indices)[0]) {PetscFree((*indices)[0]);}
855:     for (i=1; i<*nproc; i++) {
856:       if ((*indices)[i]) {PetscFree((*indices)[i]);}
857:     }
858:     PetscFree(*indices);
859:   }
860:   return(0);
861: }