Actual source code: isltog.c
1: /*$Id: isltog.c,v 1.64 2001/03/23 23:21:16 balay Exp $*/
3: #include "petscsys.h" /*I "petscsys.h" I*/
4: #include "src/vec/is/isimpl.h" /*I "petscis.h" I*/
6: /*@C
7: ISLocalToGlobalMappingGetSize - Gets the local size of a local to global mapping.
9: Not Collective
11: Input Parameter:
12: . ltog - local to global mapping
14: Output Parameter:
15: . n - the number of entries in the local mapping
17: Level: advanced
19: Concepts: mapping^local to global
21: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
22: @*/
23: int ISLocalToGlobalMappingGetSize(ISLocalToGlobalMapping mapping,int *n)
24: {
27: *n = mapping->n;
28: return(0);
29: }
31: /*@C
32: ISLocalToGlobalMappingView - View a local to global mapping
34: Not Collective
36: Input Parameters:
37: + ltog - local to global mapping
38: - viewer - viewer
40: Level: advanced
42: Concepts: mapping^local to global
44: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
45: @*/
46: int ISLocalToGlobalMappingView(ISLocalToGlobalMapping mapping,PetscViewer viewer)
47: {
48: int i,ierr,rank;
49: PetscTruth isascii;
53: if (!viewer) viewer = PETSC_VIEWER_STDOUT_(mapping->comm);
57: MPI_Comm_rank(mapping->comm,&rank);
58: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
59: if (isascii) {
60: for (i=0; i<mapping->n; i++) {
61: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %d %dn",rank,i,mapping->indices[i]);
62: }
63: PetscViewerFlush(viewer);
64: } else {
65: SETERRQ1(1,"Viewer type %s not supported for ISLocalToGlobalMapping",((PetscObject)viewer)->type_name);
66: }
68: return(0);
69: }
71: /*@C
72: ISLocalToGlobalMappingCreateIS - Creates a mapping between a local (0 to n)
73: ordering and a global parallel ordering.
75: Not collective
77: Input Parameter:
78: . is - index set containing the global numbers for each local
80: Output Parameter:
81: . mapping - new mapping data structure
83: Level: advanced
85: Concepts: mapping^local to global
87: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
88: @*/
89: int ISLocalToGlobalMappingCreateIS(IS is,ISLocalToGlobalMapping *mapping)
90: {
91: int n,*indices,ierr;
92: MPI_Comm comm;
97: PetscObjectGetComm((PetscObject)is,&comm);
98: ISGetLocalSize(is,&n);
99: ISGetIndices(is,&indices);
100: ISLocalToGlobalMappingCreate(comm,n,indices,mapping);
101: ISRestoreIndices(is,&indices);
103: return(0);
104: }
106: /*@C
107: ISLocalToGlobalMappingCreate - Creates a mapping between a local (0 to n)
108: ordering and a global parallel ordering.
110: Not Collective, but communicator may have more than one process
112: Input Parameters:
113: + comm - MPI communicator
114: . n - the number of local elements
115: - indices - the global index for each local element
117: Output Parameter:
118: . mapping - new mapping data structure
120: Level: advanced
122: Concepts: mapping^local to global
124: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS()
125: @*/
126: int ISLocalToGlobalMappingCreate(MPI_Comm cm,int n,const int indices[],ISLocalToGlobalMapping *mapping)
127: {
134: PetscHeaderCreate(*mapping,_p_ISLocalToGlobalMapping,int,IS_LTOGM_COOKIE,0,"ISLocalToGlobalMapping",
135: cm,ISLocalToGlobalMappingDestroy,ISLocalToGlobalMappingView);
136: PetscLogObjectCreate(*mapping);
137: PetscLogObjectMemory(*mapping,sizeof(struct _p_ISLocalToGlobalMapping)+n*sizeof(int));
139: (*mapping)->n = n;
140: PetscMalloc((n+1)*sizeof(int),&(*mapping)->indices);
141: PetscMemcpy((*mapping)->indices,indices,n*sizeof(int));
143: /*
144: Do not create the global to local mapping. This is only created if
145: ISGlobalToLocalMapping() is called
146: */
147: (*mapping)->globals = 0;
148: return(0);
149: }
151: /*@C
152: ISLocalToGlobalMappingBlock - Creates a blocked index version of an
153: ISLocalToGlobalMapping that is appropriate for MatSetLocalToGlobalMappingBlock()
154: and VecSetLocalToGlobalMappingBlock().
156: Not Collective, but communicator may have more than one process
158: Input Parameters:
159: + inmap - original point-wise mapping
160: - bs - block size
162: Output Parameter:
163: . outmap - block based mapping
165: Level: advanced
167: Concepts: mapping^local to global
169: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingCreateIS()
170: @*/
171: int ISLocalToGlobalMappingBlock(ISLocalToGlobalMapping inmap,int bs,ISLocalToGlobalMapping *outmap)
172: {
173: int ierr,*ii,i,n;
177: if (bs > 1) {
178: n = inmap->n/bs;
179: PetscMalloc(n*sizeof(int),&ii);
180: for (i=0; i<n; i++) {
181: ii[i] = inmap->indices[bs*i]/bs;
182: }
183: ISLocalToGlobalMappingCreate(inmap->comm,n,ii,outmap);
184: PetscFree(ii);
185: } else {
186: *outmap = inmap;
187: ierr = PetscObjectReference((PetscObject)inmap);
188: }
189: return(0);
190: }
191:
192: /*@
193: ISLocalToGlobalMappingDestroy - Destroys a mapping between a local (0 to n)
194: ordering and a global parallel ordering.
196: Note Collective
198: Input Parameters:
199: . mapping - mapping data structure
201: Level: advanced
203: .seealso: ISLocalToGlobalMappingCreate()
204: @*/
205: int ISLocalToGlobalMappingDestroy(ISLocalToGlobalMapping mapping)
206: {
210: if (--mapping->refct > 0) return(0);
211: if (mapping->refct < 0) {
212: SETERRQ(1,"Mapping already destroyed");
213: }
215: PetscFree(mapping->indices);
216: if (mapping->globals) {PetscFree(mapping->globals);}
217: PetscLogObjectDestroy(mapping);
218: PetscHeaderDestroy(mapping);
219: return(0);
220: }
221:
222: /*@
223: ISLocalToGlobalMappingApplyIS - Creates from an IS in the local numbering
224: a new index set using the global numbering defined in an ISLocalToGlobalMapping
225: context.
227: Not collective
229: Input Parameters:
230: + mapping - mapping between local and global numbering
231: - is - index set in local numbering
233: Output Parameters:
234: . newis - index set in global numbering
236: Level: advanced
238: Concepts: mapping^local to global
240: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
241: ISLocalToGlobalMappingDestroy(), ISGlobalToLocalMappingApply()
242: @*/
243: int ISLocalToGlobalMappingApplyIS(ISLocalToGlobalMapping mapping,IS is,IS *newis)
244: {
245: int ierr,n,i,*idxin,*idxmap,*idxout,Nmax = mapping->n;
252: ierr = ISGetLocalSize(is,&n);
253: ierr = ISGetIndices(is,&idxin);
254: idxmap = mapping->indices;
255:
256: PetscMalloc((n+1)*sizeof(int),&idxout);
257: for (i=0; i<n; i++) {
258: if (idxin[i] >= Nmax) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Local index %d too large %d (max) at %d",idxin[i],Nmax,i);
259: idxout[i] = idxmap[idxin[i]];
260: }
261: ISRestoreIndices(is,&idxin);
262: ISCreateGeneral(PETSC_COMM_SELF,n,idxout,newis);
263: PetscFree(idxout);
264: return(0);
265: }
267: /*MC
268: ISLocalToGlobalMappingApply - Takes a list of integers in a local numbering
269: and converts them to the global numbering.
271: Not collective
273: Input Parameters:
274: + mapping - the local to global mapping context
275: . N - number of integers
276: - in - input indices in local numbering
278: Output Parameter:
279: . out - indices in global numbering
281: Synopsis:
282: ISLocalToGlobalMappingApply(ISLocalToGlobalMapping mapping,int N,int in[],int out[])
284: Notes:
285: The in and out array parameters may be identical.
287: Level: advanced
289: .seealso: ISLocalToGlobalMappingCreate(),ISLocalToGlobalMappingDestroy(),
290: ISLocalToGlobalMappingApplyIS(),AOCreateBasic(),AOApplicationToPetsc(),
291: AOPetscToApplication(), ISGlobalToLocalMappingApply()
293: Concepts: mapping^local to global
295: M*/
297: /* -----------------------------------------------------------------------------------------*/
299: /*
300: Creates the global fields in the ISLocalToGlobalMapping structure
301: */
302: static int ISGlobalToLocalMappingSetUp_Private(ISLocalToGlobalMapping mapping)
303: {
304: int ierr,i,*idx = mapping->indices,n = mapping->n,end,start,*globals;
307: end = 0;
308: start = 100000000;
310: for (i=0; i<n; i++) {
311: if (idx[i] < 0) continue;
312: if (idx[i] < start) start = idx[i];
313: if (idx[i] > end) end = idx[i];
314: }
315: if (start > end) {start = 0; end = -1;}
316: mapping->globalstart = start;
317: mapping->globalend = end;
319: ierr = PetscMalloc((end-start+2)*sizeof(int),&globals);
320: mapping->globals = globals;
321: for (i=0; i<end-start+1; i++) {
322: globals[i] = -1;
323: }
324: for (i=0; i<n; i++) {
325: if (idx[i] < 0) continue;
326: globals[idx[i] - start] = i;
327: }
329: PetscLogObjectMemory(mapping,(end-start+1)*sizeof(int));
330: return(0);
331: }
333: /*@
334: ISGlobalToLocalMappingApply - Provides the local numbering for a list of integers
335: specified with a global numbering.
337: Not collective
339: Input Parameters:
340: + mapping - mapping between local and global numbering
341: . type - IS_GTOLM_MASK - replaces global indices with no local value with -1
342: IS_GTOLM_DROP - drops the indices with no local value from the output list
343: . n - number of global indices to map
344: - idx - global indices to map
346: Output Parameters:
347: + nout - number of indices in output array (if type == IS_GTOLM_MASK then nout = n)
348: - idxout - local index of each global index, one must pass in an array long enough
349: to hold all the indices. You can call ISGlobalToLocalMappingApply() with
350: idxout == PETSC_NULL to determine the required length (returned in nout)
351: and then allocate the required space and call ISGlobalToLocalMappingApply()
352: a second time to set the values.
354: Notes:
355: Either nout or idxout may be PETSC_NULL. idx and idxout may be identical.
357: This is not scalable in memory usage. Each processor requires O(Nglobal) size
358: array to compute these.
360: Level: advanced
362: Concepts: mapping^global to local
364: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
365: ISLocalToGlobalMappingDestroy()
366: @*/
367: int ISGlobalToLocalMappingApply(ISLocalToGlobalMapping mapping,ISGlobalToLocalMappingType type,
368: int n,const int idx[],int *nout,int idxout[])
369: {
370: int i,ierr,*globals,nf = 0,tmp,start,end;
373: if (!mapping->globals) {
374: ISGlobalToLocalMappingSetUp_Private(mapping);
375: }
376: globals = mapping->globals;
377: start = mapping->globalstart;
378: end = mapping->globalend;
380: if (type == IS_GTOLM_MASK) {
381: if (idxout) {
382: for (i=0; i<n; i++) {
383: if (idx[i] < 0) idxout[i] = idx[i];
384: else if (idx[i] < start) idxout[i] = -1;
385: else if (idx[i] > end) idxout[i] = -1;
386: else idxout[i] = globals[idx[i] - start];
387: }
388: }
389: if (nout) *nout = n;
390: } else {
391: if (idxout) {
392: for (i=0; i<n; i++) {
393: if (idx[i] < 0) continue;
394: if (idx[i] < start) continue;
395: if (idx[i] > end) continue;
396: tmp = globals[idx[i] - start];
397: if (tmp < 0) continue;
398: idxout[nf++] = tmp;
399: }
400: } else {
401: for (i=0; i<n; i++) {
402: if (idx[i] < 0) continue;
403: if (idx[i] < start) continue;
404: if (idx[i] > end) continue;
405: tmp = globals[idx[i] - start];
406: if (tmp < 0) continue;
407: nf++;
408: }
409: }
410: if (nout) *nout = nf;
411: }
413: return(0);
414: }
416: /*@C
417: ISLocalToGlobalMappingGetInfo - Gets the neighbor information for each processor and
418: each index shared by more than one processor
420: Collective on ISLocalToGlobalMapping
422: Input Parameters:
423: . mapping - the mapping from local to global indexing
425: Output Parameter:
426: + nproc - number of processors that are connected to this one
427: . proc - neighboring processors
428: . numproc - number of indices for each subdomain (processor)
429: - indices - indices of local nodes shared with neighbor (sorted by global numbering)
431: Level: advanced
433: Concepts: mapping^local to global
435: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
436: ISLocalToGlobalMappingRestoreInfo()
437: @*/
438: int ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping mapping,int *nproc,int **procs,int **numprocs,int ***indices)
439: {
440: int i,n = mapping->n,ierr,Ng,ng,max = 0,*lindices = mapping->indices;
441: int size,rank,*nprocs,*owner,nsends,*sends,j,*starts,*work,nmax,nrecvs,*recvs,proc;
442: int tag1,tag2,tag3,cnt,*len,*source,imdex,scale,*ownedsenders,*nownedsenders,rstart,nowned;
443: int node,nownedm,nt,*sends2,nsends2,*starts2,*lens2,*dest,nrecvs2,*starts3,*recvs2,k,*bprocs,*tmp;
444: int first_procs,first_numprocs,*first_indices;
445: MPI_Request *recv_waits,*send_waits;
446: MPI_Status recv_status,*send_status,*recv_statuses;
447: MPI_Comm comm = mapping->comm;
448: PetscTruth debug = PETSC_FALSE;
451: ierr = MPI_Comm_size(comm,&size);
452: ierr = MPI_Comm_rank(comm,&rank);
453: if (size == 1) {
454: *nproc = 0;
455: *procs = PETSC_NULL;
456: ierr = PetscMalloc(sizeof(int),numprocs);
457: (*numprocs)[0] = 0;
458: ierr = PetscMalloc(sizeof(int*),indices);
459: (*indices)[0] = PETSC_NULL;
460: return(0);
461: }
463: PetscOptionsHasName(PETSC_NULL,"-islocaltoglobalmappinggetinfo_debug",&debug);
465: /*
466: Notes on ISLocalToGlobalMappingGetInfo
468: globally owned node - the nodes that have been assigned to this processor in global
469: numbering, just for this routine.
471: nontrivial globally owned node - node assigned to this processor that is on a subdomain
472: boundary (i.e. is has more than one local owner)
474: locally owned node - node that exists on this processors subdomain
476: nontrivial locally owned node - node that is not in the interior (i.e. has more than one
477: local subdomain
478: */
479: PetscObjectGetNewTag((PetscObject)mapping,&tag1);
480: PetscObjectGetNewTag((PetscObject)mapping,&tag2);
481: PetscObjectGetNewTag((PetscObject)mapping,&tag3);
483: for (i=0; i<n; i++) {
484: if (lindices[i] > max) max = lindices[i];
485: }
486: ierr = MPI_Allreduce(&max,&Ng,1,MPI_INT,MPI_MAX,comm);
487: Ng++;
488: ierr = MPI_Comm_size(comm,&size);
489: ierr = MPI_Comm_rank(comm,&rank);
490: scale = Ng/size + 1;
491: ng = scale; if (rank == size-1) ng = Ng - scale*(size-1); ng = PetscMax(1,ng);
492: rstart = scale*rank;
494: /* determine ownership ranges of global indices */
495: PetscMalloc((2*size+1)*sizeof(int),&nprocs);
496: PetscMemzero(nprocs,2*size*sizeof(int));
498: /* determine owners of each local node */
499: PetscMalloc((n+1)*sizeof(int),&owner);
500: for (i=0; i<n; i++) {
501: proc = lindices[i]/scale; /* processor that globally owns this index */
502: nprocs[size+proc] = 1; /* processor globally owns at least one of ours */
503: owner[i] = proc;
504: nprocs[proc]++; /* count of how many that processor globally owns of ours */
505: }
506: nsends = 0; for (i=0; i<size; i++) nsends += nprocs[size + i];
507: PetscLogInfo(0,"ISLocalToGlobalMappingGetInfo: Number of global owners for my local data %dn",nsends);
509: /* inform other processors of number of messages and max length*/
510: PetscMalloc(2*size*sizeof(int),&work);
511: ierr = MPI_Allreduce(nprocs,work,2*size,MPI_INT,PetscMaxSum_Op,comm);
512: nmax = work[rank];
513: nrecvs = work[size+rank];
514: ierr = PetscFree(work);
515: PetscLogInfo(0,"ISLocalToGlobalMappingGetInfo: Number of local owners for my global data %dn",nrecvs);
517: /* post receives for owned rows */
518: PetscMalloc((2*nrecvs+1)*(nmax+1)*sizeof(int),&recvs);
519: PetscMalloc((nrecvs+1)*sizeof(MPI_Request),&recv_waits);
520: for (i=0; i<nrecvs; i++) {
521: MPI_Irecv(recvs+2*nmax*i,2*nmax,MPI_INT,MPI_ANY_SOURCE,tag1,comm,recv_waits+i);
522: }
524: /* pack messages containing lists of local nodes to owners */
525: ierr = PetscMalloc((2*n+1)*sizeof(int),&sends);
526: ierr = PetscMalloc((size+1)*sizeof(int),&starts);
527: starts[0] = 0;
528: for (i=1; i<size; i++) { starts[i] = starts[i-1] + 2*nprocs[i-1];}
529: for (i=0; i<n; i++) {
530: sends[starts[owner[i]]++] = lindices[i];
531: sends[starts[owner[i]]++] = i;
532: }
533: PetscFree(owner);
534: starts[0] = 0;
535: for (i=1; i<size; i++) { starts[i] = starts[i-1] + 2*nprocs[i-1];}
537: /* send the messages */
538: PetscMalloc((nsends+1)*sizeof(MPI_Request),&send_waits);
539: PetscMalloc((nsends+1)*sizeof(int),&dest);
540: cnt = 0;
541: for (i=0; i<size; i++) {
542: if (nprocs[i]) {
543: ierr = MPI_Isend(sends+starts[i],2*nprocs[i],MPI_INT,i,tag1,comm,send_waits+cnt);
544: dest[cnt] = i;
545: cnt++;
546: }
547: }
548: PetscFree(starts);
550: /* wait on receives */
551: PetscMalloc((2*nrecvs+1)*sizeof(int),&source);
552: len = source + nrecvs;
553: cnt = nrecvs;
554: PetscMalloc((ng+1)*sizeof(int),&nownedsenders);
555: PetscMemzero(nownedsenders,ng*sizeof(int));
556: while (cnt) {
557: MPI_Waitany(nrecvs,recv_waits,&imdex,&recv_status);
558: /* unpack receives into our local space */
559: ierr = MPI_Get_count(&recv_status,MPI_INT,&len[imdex]);
560: source[imdex] = recv_status.MPI_SOURCE;
561: len[imdex] = len[imdex]/2;
562: /* count how many local owners for each of my global owned indices */
563: for (i=0; i<len[imdex]; i++) nownedsenders[recvs[2*imdex*nmax+2*i]-rstart]++;
564: cnt--;
565: }
566: PetscFree(recv_waits);
568: /* count how many globally owned indices are on an edge multiplied by how many processors own them. */
569: nowned = 0;
570: nownedm = 0;
571: for (i=0; i<ng; i++) {
572: if (nownedsenders[i] > 1) {nownedm += nownedsenders[i]; nowned++;}
573: }
575: /* create single array to contain rank of all local owners of each globally owned index */
576: ierr = PetscMalloc((nownedm+1)*sizeof(int),&ownedsenders);
577: ierr = PetscMalloc((ng+1)*sizeof(int),&starts);
578: starts[0] = 0;
579: for (i=1; i<ng; i++) {
580: if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
581: else starts[i] = starts[i-1];
582: }
584: /* for each nontrival globally owned node list all arriving processors */
585: for (i=0; i<nrecvs; i++) {
586: for (j=0; j<len[i]; j++) {
587: node = recvs[2*i*nmax+2*j]-rstart;
588: if (nownedsenders[node] > 1) {
589: ownedsenders[starts[node]++] = source[i];
590: }
591: }
592: }
594: if (debug) { /* ----------------------------------- */
595: starts[0] = 0;
596: for (i=1; i<ng; i++) {
597: if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
598: else starts[i] = starts[i-1];
599: }
600: for (i=0; i<ng; i++) {
601: if (nownedsenders[i] > 1) {
602: PetscSynchronizedPrintf(comm,"[%d] global node %d local owner processors: ",rank,i+rstart);
603: for (j=0; j<nownedsenders[i]; j++) {
604: PetscSynchronizedPrintf(comm,"%d ",ownedsenders[starts[i]+j]);
605: }
606: PetscSynchronizedPrintf(comm,"n");
607: }
608: }
609: PetscSynchronizedFlush(comm);
610: }/* ----------------------------------- */
612: /* wait on original sends */
613: if (nsends) {
614: PetscMalloc(nsends*sizeof(MPI_Status),&send_status);
615: MPI_Waitall(nsends,send_waits,send_status);
616: PetscFree(send_status);
617: }
618: PetscFree(send_waits);
619: PetscFree(sends);
620: PetscFree(nprocs);
622: /* pack messages to send back to local owners */
623: starts[0] = 0;
624: for (i=1; i<ng; i++) {
625: if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
626: else starts[i] = starts[i-1];
627: }
628: nsends2 = nrecvs;
629: ierr = PetscMalloc((nsends2+1)*sizeof(int),&nprocs); /* length of each message */
630: for (i=0; i<nrecvs; i++) {
631: nprocs[i] = 1;
632: for (j=0; j<len[i]; j++) {
633: node = recvs[2*i*nmax+2*j]-rstart;
634: if (nownedsenders[node] > 1) {
635: nprocs[i] += 2 + nownedsenders[node];
636: }
637: }
638: }
639: nt = 0; for (i=0; i<nsends2; i++) nt += nprocs[i];
640: PetscMalloc((nt+1)*sizeof(int),&sends2);
641: PetscMalloc((nsends2+1)*sizeof(int),&starts2);
642: starts2[0] = 0; for (i=1; i<nsends2; i++) starts2[i] = starts2[i-1] + nprocs[i-1];
643: /*
644: Each message is 1 + nprocs[i] long, and consists of
645: (0) the number of nodes being sent back
646: (1) the local node number,
647: (2) the number of processors sharing it,
648: (3) the processors sharing it
649: */
650: for (i=0; i<nsends2; i++) {
651: cnt = 1;
652: sends2[starts2[i]] = 0;
653: for (j=0; j<len[i]; j++) {
654: node = recvs[2*i*nmax+2*j]-rstart;
655: if (nownedsenders[node] > 1) {
656: sends2[starts2[i]]++;
657: sends2[starts2[i]+cnt++] = recvs[2*i*nmax+2*j+1];
658: sends2[starts2[i]+cnt++] = nownedsenders[node];
659: PetscMemcpy(&sends2[starts2[i]+cnt],&ownedsenders[starts[node]],nownedsenders[node]*sizeof(int));
660: cnt += nownedsenders[node];
661: }
662: }
663: }
665: /* send the message lengths */
666: for (i=0; i<nsends2; i++) {
667: MPI_Send(&nprocs[i],1,MPI_INT,source[i],tag2,comm);
668: }
670: /* receive the message lengths */
671: nrecvs2 = nsends;
672: PetscMalloc((nrecvs2+1)*sizeof(int),&lens2);
673: PetscMalloc((nrecvs2+1)*sizeof(int),&starts3);
674: nt = 0;
675: for (i=0; i<nrecvs2; i++) {
676: MPI_Recv(&lens2[i],1,MPI_INT,dest[i],tag2,comm,&recv_status);
677: nt += lens2[i];
678: }
679: starts3[0] = 0;
680: for (i=0; i<nrecvs2-1; i++) {
681: starts3[i+1] = starts3[i] + lens2[i];
682: }
683: PetscMalloc((nt+1)*sizeof(int),&recvs2);
684: PetscMalloc((nrecvs2+1)*sizeof(MPI_Request),&recv_waits);
685: for (i=0; i<nrecvs2; i++) {
686: MPI_Irecv(recvs2+starts3[i],lens2[i],MPI_INT,dest[i],tag3,comm,recv_waits+i);
687: }
688:
689: /* send the messages */
690: PetscMalloc((nsends2+1)*sizeof(MPI_Request),&send_waits);
691: for (i=0; i<nsends2; i++) {
692: MPI_Isend(sends2+starts2[i],nprocs[i],MPI_INT,source[i],tag3,comm,send_waits+i);
693: }
695: /* wait on receives */
696: PetscMalloc((nrecvs2+1)*sizeof(MPI_Status),&recv_statuses);
697: MPI_Waitall(nrecvs2,recv_waits,recv_statuses);
698: PetscFree(recv_statuses);
699: PetscFree(recv_waits);
700: PetscFree(nprocs);
702: if (debug) { /* ----------------------------------- */
703: cnt = 0;
704: for (i=0; i<nrecvs2; i++) {
705: nt = recvs2[cnt++];
706: for (j=0; j<nt; j++) {
707: PetscSynchronizedPrintf(comm,"[%d] local node %d number of subdomains %d: ",rank,recvs2[cnt],recvs2[cnt+1]);
708: for (k=0; k<recvs2[cnt+1]; k++) {
709: PetscSynchronizedPrintf(comm,"%d ",recvs2[cnt+2+k]);
710: }
711: cnt += 2 + recvs2[cnt+1];
712: PetscSynchronizedPrintf(comm,"n");
713: }
714: }
715: PetscSynchronizedFlush(comm);
716: } /* ----------------------------------- */
718: /* count number subdomains for each local node */
719: PetscMalloc(size*sizeof(int),&nprocs);
720: PetscMemzero(nprocs,size*sizeof(int));
721: cnt = 0;
722: for (i=0; i<nrecvs2; i++) {
723: nt = recvs2[cnt++];
724: for (j=0; j<nt; j++) {
725: for (k=0; k<recvs2[cnt+1]; k++) {
726: nprocs[recvs2[cnt+2+k]]++;
727: }
728: cnt += 2 + recvs2[cnt+1];
729: }
730: }
731: nt = 0; for (i=0; i<size; i++) nt += (nprocs[i] > 0);
732: *nproc = nt;
733: PetscMalloc((nt+1)*sizeof(int),procs);
734: PetscMalloc((nt+1)*sizeof(int),numprocs);
735: PetscMalloc((nt+1)*sizeof(int*),indices);
736: PetscMalloc(size*sizeof(int),&bprocs);
737: cnt = 0;
738: for (i=0; i<size; i++) {
739: if (nprocs[i] > 0) {
740: bprocs[i] = cnt;
741: (*procs)[cnt] = i;
742: (*numprocs)[cnt] = nprocs[i];
743: ierr = PetscMalloc(nprocs[i]*sizeof(int),&(*indices)[cnt]);
744: cnt++;
745: }
746: }
748: /* make the list of subdomains for each nontrivial local node */
749: PetscMemzero(*numprocs,nt*sizeof(int));
750: cnt = 0;
751: for (i=0; i<nrecvs2; i++) {
752: nt = recvs2[cnt++];
753: for (j=0; j<nt; j++) {
754: for (k=0; k<recvs2[cnt+1]; k++) {
755: (*indices)[bprocs[recvs2[cnt+2+k]]][(*numprocs)[bprocs[recvs2[cnt+2+k]]]++] = recvs2[cnt];
756: }
757: cnt += 2 + recvs2[cnt+1];
758: }
759: }
760: PetscFree(bprocs);
761: PetscFree(recvs2);
763: /* sort the node indexing by their global numbers */
764: nt = *nproc;
765: for (i=0; i<nt; i++) {
766: PetscMalloc(((*numprocs)[i])*sizeof(int),&tmp);
767: for (j=0; j<(*numprocs)[i]; j++) {
768: tmp[j] = lindices[(*indices)[i][j]];
769: }
770: PetscSortIntWithArray((*numprocs)[i],tmp,(*indices)[i]);
771: PetscFree(tmp);
772: }
774: if (debug) { /* ----------------------------------- */
775: nt = *nproc;
776: for (i=0; i<nt; i++) {
777: PetscSynchronizedPrintf(comm,"[%d] subdomain %d number of indices %d: ",rank,(*procs)[i],(*numprocs)[i]);
778: for (j=0; j<(*numprocs)[i]; j++) {
779: PetscSynchronizedPrintf(comm,"%d ",(*indices)[i][j]);
780: }
781: PetscSynchronizedPrintf(comm,"n");
782: }
783: PetscSynchronizedFlush(comm);
784: } /* ----------------------------------- */
786: /* wait on sends */
787: if (nsends2) {
788: PetscMalloc(nsends2*sizeof(MPI_Status),&send_status);
789: MPI_Waitall(nsends2,send_waits,send_status);
790: PetscFree(send_status);
791: }
793: PetscFree(starts3);
794: PetscFree(dest);
795: PetscFree(send_waits);
797: PetscFree(nownedsenders);
798: PetscFree(ownedsenders);
799: PetscFree(starts);
800: PetscFree(starts2);
801: PetscFree(lens2);
803: PetscFree(source);
804: PetscFree(recvs);
805: PetscFree(nprocs);
806: PetscFree(sends2);
808: /* put the information about myself as the first entry in the list */
809: first_procs = (*procs)[0];
810: first_numprocs = (*numprocs)[0];
811: first_indices = (*indices)[0];
812: for (i=0; i<*nproc; i++) {
813: if ((*procs)[i] == rank) {
814: (*procs)[0] = (*procs)[i];
815: (*numprocs)[0] = (*numprocs)[i];
816: (*indices)[0] = (*indices)[i];
817: (*procs)[i] = first_procs;
818: (*numprocs)[i] = first_numprocs;
819: (*indices)[i] = first_indices;
820: break;
821: }
822: }
824: return(0);
825: }
827: /*@C
828: ISLocalToGlobalMappingRestoreInfo - Frees the memory allocated by ISLocalToGlobalMappingGetInfo()
830: Collective on ISLocalToGlobalMapping
832: Input Parameters:
833: . mapping - the mapping from local to global indexing
835: Output Parameter:
836: + nproc - number of processors that are connected to this one
837: . proc - neighboring processors
838: . numproc - number of indices for each processor
839: - indices - indices of local nodes shared with neighbor (sorted by global numbering)
841: Level: advanced
843: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
844: ISLocalToGlobalMappingGetInfo()
845: @*/
846: int ISLocalToGlobalMappingRestoreInfo(ISLocalToGlobalMapping mapping,int *nproc,int **procs,int **numprocs,int ***indices)
847: {
848: int ierr,i;
851: if (*procs) {PetscFree(*procs);}
852: if (*numprocs) {PetscFree(*numprocs);}
853: if (*indices) {
854: if ((*indices)[0]) {PetscFree((*indices)[0]);}
855: for (i=1; i<*nproc; i++) {
856: if ((*indices)[i]) {PetscFree((*indices)[i]);}
857: }
858: PetscFree(*indices);
859: }
860: return(0);
861: }