Actual source code: isltog.c
1: #define PETSCVEC_DLL
3: #include petscvec.h
4: #include include/private/isimpl.h
6: PetscCookie IS_LTOGM_COOKIE = 0;
10: /*@C
11: ISLocalToGlobalMappingGetSize - Gets the local size of a local to global mapping.
13: Not Collective
15: Input Parameter:
16: . ltog - local to global mapping
18: Output Parameter:
19: . n - the number of entries in the local mapping
21: Level: advanced
23: Concepts: mapping^local to global
25: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
26: @*/
27: PetscErrorCode ISLocalToGlobalMappingGetSize(ISLocalToGlobalMapping mapping,PetscInt *n)
28: {
32: *n = mapping->n;
33: return(0);
34: }
38: /*@C
39: ISLocalToGlobalMappingView - View a local to global mapping
41: Not Collective
43: Input Parameters:
44: + ltog - local to global mapping
45: - viewer - viewer
47: Level: advanced
49: Concepts: mapping^local to global
51: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
52: @*/
53: PetscErrorCode ISLocalToGlobalMappingView(ISLocalToGlobalMapping mapping,PetscViewer viewer)
54: {
55: PetscInt i;
56: PetscMPIInt rank;
57: PetscTruth iascii;
58: PetscErrorCode ierr;
62: if (!viewer) {
63: PetscViewerASCIIGetStdout(((PetscObject)mapping)->comm,&viewer);
64: }
67: MPI_Comm_rank(((PetscObject)mapping)->comm,&rank);
68: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
69: if (iascii) {
70: for (i=0; i<mapping->n; i++) {
71: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %d %d\n",rank,i,mapping->indices[i]);
72: }
73: PetscViewerFlush(viewer);
74: } else {
75: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISLocalToGlobalMapping",((PetscObject)viewer)->type_name);
76: }
78: return(0);
79: }
83: /*@
84: ISLocalToGlobalMappingCreateIS - Creates a mapping between a local (0 to n)
85: ordering and a global parallel ordering.
87: Not collective
89: Input Parameter:
90: . is - index set containing the global numbers for each local
92: Output Parameter:
93: . mapping - new mapping data structure
95: Level: advanced
97: Concepts: mapping^local to global
99: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
100: @*/
101: PetscErrorCode ISLocalToGlobalMappingCreateIS(IS is,ISLocalToGlobalMapping *mapping)
102: {
104: PetscInt n,*indices;
105: MPI_Comm comm;
111: PetscObjectGetComm((PetscObject)is,&comm);
112: ISGetLocalSize(is,&n);
113: ISGetIndices(is,&indices);
114: ISLocalToGlobalMappingCreate(comm,n,indices,mapping);
115: ISRestoreIndices(is,&indices);
117: return(0);
118: }
123: /*@
124: ISLocalToGlobalMappingCreate - Creates a mapping between a local (0 to n)
125: ordering and a global parallel ordering.
127: Not Collective, but communicator may have more than one process
129: Input Parameters:
130: + comm - MPI communicator
131: . n - the number of local elements
132: - indices - the global index for each local element
134: Output Parameter:
135: . mapping - new mapping data structure
137: Level: advanced
139: Concepts: mapping^local to global
141: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreateNC()
142: @*/
143: PetscErrorCode ISLocalToGlobalMappingCreate(MPI_Comm cm,PetscInt n,const PetscInt indices[],ISLocalToGlobalMapping *mapping)
144: {
146: PetscInt *in;
151: PetscMalloc(n*sizeof(PetscInt),&in);
152: PetscMemcpy(in,indices,n*sizeof(PetscInt));
153: ISLocalToGlobalMappingCreateNC(cm,n,in,mapping);
154: return(0);
155: }
159: /*@C
160: ISLocalToGlobalMappingCreateNC - Creates a mapping between a local (0 to n)
161: ordering and a global parallel ordering.
163: Not Collective, but communicator may have more than one process
165: Input Parameters:
166: + comm - MPI communicator
167: . n - the number of local elements
168: - indices - the global index for each local element
170: Output Parameter:
171: . mapping - new mapping data structure
173: Level: developer
175: Notes: Does not copy the indices, just keeps the pointer to the indices. The ISLocalToGlobalMappingDestroy()
176: will free the space so it must be obtained with PetscMalloc() and it must not be freed elsewhere.
178: Concepts: mapping^local to global
180: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate()
181: @*/
182: PetscErrorCode ISLocalToGlobalMappingCreateNC(MPI_Comm cm,PetscInt n,const PetscInt indices[],ISLocalToGlobalMapping *mapping)
183: {
189: *mapping = PETSC_NULL;
190: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
191: ISInitializePackage(PETSC_NULL);
192: #endif
194: PetscHeaderCreate(*mapping,_p_ISLocalToGlobalMapping,int,IS_LTOGM_COOKIE,0,"ISLocalToGlobalMapping",
195: cm,ISLocalToGlobalMappingDestroy,ISLocalToGlobalMappingView);
197: (*mapping)->n = n;
198: (*mapping)->indices = (PetscInt*)indices;
199: PetscLogObjectMemory(*mapping,n*sizeof(PetscInt));
201: /*
202: Do not create the global to local mapping. This is only created if
203: ISGlobalToLocalMapping() is called
204: */
205: (*mapping)->globals = 0;
206: return(0);
207: }
211: /*@
212: ISLocalToGlobalMappingBlock - Creates a blocked index version of an
213: ISLocalToGlobalMapping that is appropriate for MatSetLocalToGlobalMappingBlock()
214: and VecSetLocalToGlobalMappingBlock().
216: Not Collective, but communicator may have more than one process
218: Input Parameters:
219: + inmap - original point-wise mapping
220: - bs - block size
222: Output Parameter:
223: . outmap - block based mapping; the indices are relative to BLOCKS, not individual vector or matrix entries.
225: Level: advanced
227: Concepts: mapping^local to global
229: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingCreateIS()
230: @*/
231: PetscErrorCode ISLocalToGlobalMappingBlock(ISLocalToGlobalMapping inmap,PetscInt bs,ISLocalToGlobalMapping *outmap)
232: {
234: PetscInt *ii,i,n;
239: if (bs > 1) {
240: n = inmap->n/bs;
241: if (n*bs != inmap->n) SETERRQ(PETSC_ERR_ARG_INCOMP,"Pointwise mapping length is not divisible by block size");
242: PetscMalloc(n*sizeof(PetscInt),&ii);
243: for (i=0; i<n; i++) {
244: ii[i] = inmap->indices[bs*i]/bs;
245: }
246: ISLocalToGlobalMappingCreate(((PetscObject)inmap)->comm,n,ii,outmap);
247: PetscFree(ii);
248: } else {
249: PetscObjectReference((PetscObject)inmap);
250: *outmap = inmap;
251: }
252: return(0);
253: }
254:
257: /*@
258: ISLocalToGlobalMappingDestroy - Destroys a mapping between a local (0 to n)
259: ordering and a global parallel ordering.
261: Note Collective
263: Input Parameters:
264: . mapping - mapping data structure
266: Level: advanced
268: .seealso: ISLocalToGlobalMappingCreate()
269: @*/
270: PetscErrorCode ISLocalToGlobalMappingDestroy(ISLocalToGlobalMapping mapping)
271: {
275: if (--((PetscObject)mapping)->refct > 0) return(0);
276: PetscFree(mapping->indices);
277: PetscFree(mapping->globals);
278: PetscHeaderDestroy(mapping);
279: return(0);
280: }
281:
284: /*@
285: ISLocalToGlobalMappingApplyIS - Creates from an IS in the local numbering
286: a new index set using the global numbering defined in an ISLocalToGlobalMapping
287: context.
289: Not collective
291: Input Parameters:
292: + mapping - mapping between local and global numbering
293: - is - index set in local numbering
295: Output Parameters:
296: . newis - index set in global numbering
298: Level: advanced
300: Concepts: mapping^local to global
302: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
303: ISLocalToGlobalMappingDestroy(), ISGlobalToLocalMappingApply()
304: @*/
305: PetscErrorCode ISLocalToGlobalMappingApplyIS(ISLocalToGlobalMapping mapping,IS is,IS *newis)
306: {
308: PetscInt n,i,*idxin,*idxmap,*idxout,Nmax = mapping->n;
315: ISGetLocalSize(is,&n);
316: ISGetIndices(is,&idxin);
317: idxmap = mapping->indices;
318:
319: PetscMalloc(n*sizeof(PetscInt),&idxout);
320: for (i=0; i<n; i++) {
321: if (idxin[i] >= Nmax) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Local index %d too large %d (max) at %d",idxin[i],Nmax-1,i);
322: idxout[i] = idxmap[idxin[i]];
323: }
324: ISRestoreIndices(is,&idxin);
325: ISCreateGeneralNC(PETSC_COMM_SELF,n,idxout,newis);
326: return(0);
327: }
329: /*MC
330: ISLocalToGlobalMappingApply - Takes a list of integers in a local numbering
331: and converts them to the global numbering.
333: Not collective
335: Input Parameters:
336: + mapping - the local to global mapping context
337: . N - number of integers
338: - in - input indices in local numbering
340: Output Parameter:
341: . out - indices in global numbering
343: Synopsis:
344: PetscErrorCode ISLocalToGlobalMappingApply(ISLocalToGlobalMapping mapping,int N,int in[],int out[])
346: Notes:
347: The in and out array parameters may be identical.
349: Level: advanced
351: .seealso: ISLocalToGlobalMappingCreate(),ISLocalToGlobalMappingDestroy(),
352: ISLocalToGlobalMappingApplyIS(),AOCreateBasic(),AOApplicationToPetsc(),
353: AOPetscToApplication(), ISGlobalToLocalMappingApply()
355: Concepts: mapping^local to global
357: M*/
359: /* -----------------------------------------------------------------------------------------*/
363: /*
364: Creates the global fields in the ISLocalToGlobalMapping structure
365: */
366: static PetscErrorCode ISGlobalToLocalMappingSetUp_Private(ISLocalToGlobalMapping mapping)
367: {
369: PetscInt i,*idx = mapping->indices,n = mapping->n,end,start,*globals;
372: end = 0;
373: start = 100000000;
375: for (i=0; i<n; i++) {
376: if (idx[i] < 0) continue;
377: if (idx[i] < start) start = idx[i];
378: if (idx[i] > end) end = idx[i];
379: }
380: if (start > end) {start = 0; end = -1;}
381: mapping->globalstart = start;
382: mapping->globalend = end;
384: PetscMalloc((end-start+2)*sizeof(PetscInt),&globals);
385: mapping->globals = globals;
386: for (i=0; i<end-start+1; i++) {
387: globals[i] = -1;
388: }
389: for (i=0; i<n; i++) {
390: if (idx[i] < 0) continue;
391: globals[idx[i] - start] = i;
392: }
394: PetscLogObjectMemory(mapping,(end-start+1)*sizeof(PetscInt));
395: return(0);
396: }
400: /*@
401: ISGlobalToLocalMappingApply - Provides the local numbering for a list of integers
402: specified with a global numbering.
404: Not collective
406: Input Parameters:
407: + mapping - mapping between local and global numbering
408: . type - IS_GTOLM_MASK - replaces global indices with no local value with -1
409: IS_GTOLM_DROP - drops the indices with no local value from the output list
410: . n - number of global indices to map
411: - idx - global indices to map
413: Output Parameters:
414: + nout - number of indices in output array (if type == IS_GTOLM_MASK then nout = n)
415: - idxout - local index of each global index, one must pass in an array long enough
416: to hold all the indices. You can call ISGlobalToLocalMappingApply() with
417: idxout == PETSC_NULL to determine the required length (returned in nout)
418: and then allocate the required space and call ISGlobalToLocalMappingApply()
419: a second time to set the values.
421: Notes:
422: Either nout or idxout may be PETSC_NULL. idx and idxout may be identical.
424: This is not scalable in memory usage. Each processor requires O(Nglobal) size
425: array to compute these.
427: Level: advanced
429: Concepts: mapping^global to local
431: .seealso: ISLocalToGlobalMappingApply(), ISLocalToGlobalMappingCreate(),
432: ISLocalToGlobalMappingDestroy()
433: @*/
434: PetscErrorCode ISGlobalToLocalMappingApply(ISLocalToGlobalMapping mapping,ISGlobalToLocalMappingType type,
435: PetscInt n,const PetscInt idx[],PetscInt *nout,PetscInt idxout[])
436: {
437: PetscInt i,*globals,nf = 0,tmp,start,end;
442: if (!mapping->globals) {
443: ISGlobalToLocalMappingSetUp_Private(mapping);
444: }
445: globals = mapping->globals;
446: start = mapping->globalstart;
447: end = mapping->globalend;
449: if (type == IS_GTOLM_MASK) {
450: if (idxout) {
451: for (i=0; i<n; i++) {
452: if (idx[i] < 0) idxout[i] = idx[i];
453: else if (idx[i] < start) idxout[i] = -1;
454: else if (idx[i] > end) idxout[i] = -1;
455: else idxout[i] = globals[idx[i] - start];
456: }
457: }
458: if (nout) *nout = n;
459: } else {
460: if (idxout) {
461: for (i=0; i<n; i++) {
462: if (idx[i] < 0) continue;
463: if (idx[i] < start) continue;
464: if (idx[i] > end) continue;
465: tmp = globals[idx[i] - start];
466: if (tmp < 0) continue;
467: idxout[nf++] = tmp;
468: }
469: } else {
470: for (i=0; i<n; i++) {
471: if (idx[i] < 0) continue;
472: if (idx[i] < start) continue;
473: if (idx[i] > end) continue;
474: tmp = globals[idx[i] - start];
475: if (tmp < 0) continue;
476: nf++;
477: }
478: }
479: if (nout) *nout = nf;
480: }
482: return(0);
483: }
487: /*@C
488: ISLocalToGlobalMappingGetInfo - Gets the neighbor information for each processor and
489: each index shared by more than one processor
491: Collective on ISLocalToGlobalMapping
493: Input Parameters:
494: . mapping - the mapping from local to global indexing
496: Output Parameter:
497: + nproc - number of processors that are connected to this one
498: . proc - neighboring processors
499: . numproc - number of indices for each subdomain (processor)
500: - indices - indices of local nodes shared with neighbor (sorted by global numbering)
502: Level: advanced
504: Concepts: mapping^local to global
506: Fortran Usage:
507: $ ISLocalToGlobalMpngGetInfoSize(ISLocalToGlobalMapping,PetscInt nproc,PetscInt numprocmax,ierr) followed by
508: $ ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping,PetscInt nproc, PetscInt procs[nproc],PetscInt numprocs[nproc],
509: PetscInt indices[nproc][numprocmax],ierr)
510: There is no ISLocalToGlobalMappingRestoreInfo() in Fortran. You must make sure that procs[], numprocs[] and
511: indices[][] are large enough arrays, either by allocating them dynamically or defining static ones large enough.
514: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
515: ISLocalToGlobalMappingRestoreInfo()
516: @*/
517: PetscErrorCode ISLocalToGlobalMappingGetInfo(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
518: {
520: PetscMPIInt size,rank,tag1,tag2,tag3,*len,*source,imdex;
521: PetscInt i,n = mapping->n,Ng,ng,max = 0,*lindices = mapping->indices;
522: PetscInt *nprocs,*owner,nsends,*sends,j,*starts,nmax,nrecvs,*recvs,proc;
523: PetscInt cnt,scale,*ownedsenders,*nownedsenders,rstart,nowned;
524: PetscInt node,nownedm,nt,*sends2,nsends2,*starts2,*lens2,*dest,nrecvs2,*starts3,*recvs2,k,*bprocs,*tmp;
525: PetscInt first_procs,first_numprocs,*first_indices;
526: MPI_Request *recv_waits,*send_waits;
527: MPI_Status recv_status,*send_status,*recv_statuses;
528: MPI_Comm comm = ((PetscObject)mapping)->comm;
529: PetscTruth debug = PETSC_FALSE;
533: MPI_Comm_size(comm,&size);
534: MPI_Comm_rank(comm,&rank);
535: if (size == 1) {
536: *nproc = 0;
537: *procs = PETSC_NULL;
538: PetscMalloc(sizeof(PetscInt),numprocs);
539: (*numprocs)[0] = 0;
540: PetscMalloc(sizeof(PetscInt*),indices);
541: (*indices)[0] = PETSC_NULL;
542: return(0);
543: }
545: PetscOptionsHasName(PETSC_NULL,"-islocaltoglobalmappinggetinfo_debug",&debug);
547: /*
548: Notes on ISLocalToGlobalMappingGetInfo
550: globally owned node - the nodes that have been assigned to this processor in global
551: numbering, just for this routine.
553: nontrivial globally owned node - node assigned to this processor that is on a subdomain
554: boundary (i.e. is has more than one local owner)
556: locally owned node - node that exists on this processors subdomain
558: nontrivial locally owned node - node that is not in the interior (i.e. has more than one
559: local subdomain
560: */
561: PetscObjectGetNewTag((PetscObject)mapping,&tag1);
562: PetscObjectGetNewTag((PetscObject)mapping,&tag2);
563: PetscObjectGetNewTag((PetscObject)mapping,&tag3);
565: for (i=0; i<n; i++) {
566: if (lindices[i] > max) max = lindices[i];
567: }
568: MPI_Allreduce(&max,&Ng,1,MPIU_INT,MPI_MAX,comm);
569: Ng++;
570: MPI_Comm_size(comm,&size);
571: MPI_Comm_rank(comm,&rank);
572: scale = Ng/size + 1;
573: ng = scale; if (rank == size-1) ng = Ng - scale*(size-1); ng = PetscMax(1,ng);
574: rstart = scale*rank;
576: /* determine ownership ranges of global indices */
577: PetscMalloc(2*size*sizeof(PetscInt),&nprocs);
578: PetscMemzero(nprocs,2*size*sizeof(PetscInt));
580: /* determine owners of each local node */
581: PetscMalloc(n*sizeof(PetscInt),&owner);
582: for (i=0; i<n; i++) {
583: proc = lindices[i]/scale; /* processor that globally owns this index */
584: nprocs[2*proc+1] = 1; /* processor globally owns at least one of ours */
585: owner[i] = proc;
586: nprocs[2*proc]++; /* count of how many that processor globally owns of ours */
587: }
588: nsends = 0; for (i=0; i<size; i++) nsends += nprocs[2*i+1];
589: PetscInfo1(mapping,"Number of global owners for my local data %d\n",nsends);
591: /* inform other processors of number of messages and max length*/
592: PetscMaxSum(comm,nprocs,&nmax,&nrecvs);
593: PetscInfo1(mapping,"Number of local owners for my global data %d\n",nrecvs);
595: /* post receives for owned rows */
596: PetscMalloc((2*nrecvs+1)*(nmax+1)*sizeof(PetscInt),&recvs);
597: PetscMalloc((nrecvs+1)*sizeof(MPI_Request),&recv_waits);
598: for (i=0; i<nrecvs; i++) {
599: MPI_Irecv(recvs+2*nmax*i,2*nmax,MPIU_INT,MPI_ANY_SOURCE,tag1,comm,recv_waits+i);
600: }
602: /* pack messages containing lists of local nodes to owners */
603: PetscMalloc((2*n+1)*sizeof(PetscInt),&sends);
604: PetscMalloc((size+1)*sizeof(PetscInt),&starts);
605: starts[0] = 0;
606: for (i=1; i<size; i++) { starts[i] = starts[i-1] + 2*nprocs[2*i-2];}
607: for (i=0; i<n; i++) {
608: sends[starts[owner[i]]++] = lindices[i];
609: sends[starts[owner[i]]++] = i;
610: }
611: PetscFree(owner);
612: starts[0] = 0;
613: for (i=1; i<size; i++) { starts[i] = starts[i-1] + 2*nprocs[2*i-2];}
615: /* send the messages */
616: PetscMalloc((nsends+1)*sizeof(MPI_Request),&send_waits);
617: PetscMalloc((nsends+1)*sizeof(PetscInt),&dest);
618: cnt = 0;
619: for (i=0; i<size; i++) {
620: if (nprocs[2*i]) {
621: MPI_Isend(sends+starts[i],2*nprocs[2*i],MPIU_INT,i,tag1,comm,send_waits+cnt);
622: dest[cnt] = i;
623: cnt++;
624: }
625: }
626: PetscFree(starts);
628: /* wait on receives */
629: PetscMalloc((nrecvs+1)*sizeof(PetscMPIInt),&source);
630: PetscMalloc((nrecvs+1)*sizeof(PetscMPIInt),&len);
631: cnt = nrecvs;
632: PetscMalloc((ng+1)*sizeof(PetscInt),&nownedsenders);
633: PetscMemzero(nownedsenders,ng*sizeof(PetscInt));
634: while (cnt) {
635: MPI_Waitany(nrecvs,recv_waits,&imdex,&recv_status);
636: /* unpack receives into our local space */
637: MPI_Get_count(&recv_status,MPIU_INT,&len[imdex]);
638: source[imdex] = recv_status.MPI_SOURCE;
639: len[imdex] = len[imdex]/2;
640: /* count how many local owners for each of my global owned indices */
641: for (i=0; i<len[imdex]; i++) nownedsenders[recvs[2*imdex*nmax+2*i]-rstart]++;
642: cnt--;
643: }
644: PetscFree(recv_waits);
646: /* count how many globally owned indices are on an edge multiplied by how many processors own them. */
647: nowned = 0;
648: nownedm = 0;
649: for (i=0; i<ng; i++) {
650: if (nownedsenders[i] > 1) {nownedm += nownedsenders[i]; nowned++;}
651: }
653: /* create single array to contain rank of all local owners of each globally owned index */
654: PetscMalloc((nownedm+1)*sizeof(PetscInt),&ownedsenders);
655: PetscMalloc((ng+1)*sizeof(PetscInt),&starts);
656: starts[0] = 0;
657: for (i=1; i<ng; i++) {
658: if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
659: else starts[i] = starts[i-1];
660: }
662: /* for each nontrival globally owned node list all arriving processors */
663: for (i=0; i<nrecvs; i++) {
664: for (j=0; j<len[i]; j++) {
665: node = recvs[2*i*nmax+2*j]-rstart;
666: if (nownedsenders[node] > 1) {
667: ownedsenders[starts[node]++] = source[i];
668: }
669: }
670: }
672: if (debug) { /* ----------------------------------- */
673: starts[0] = 0;
674: for (i=1; i<ng; i++) {
675: if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
676: else starts[i] = starts[i-1];
677: }
678: for (i=0; i<ng; i++) {
679: if (nownedsenders[i] > 1) {
680: PetscSynchronizedPrintf(comm,"[%d] global node %d local owner processors: ",rank,i+rstart);
681: for (j=0; j<nownedsenders[i]; j++) {
682: PetscSynchronizedPrintf(comm,"%d ",ownedsenders[starts[i]+j]);
683: }
684: PetscSynchronizedPrintf(comm,"\n");
685: }
686: }
687: PetscSynchronizedFlush(comm);
688: }/* ----------------------------------- */
690: /* wait on original sends */
691: if (nsends) {
692: PetscMalloc(nsends*sizeof(MPI_Status),&send_status);
693: MPI_Waitall(nsends,send_waits,send_status);
694: PetscFree(send_status);
695: }
696: PetscFree(send_waits);
697: PetscFree(sends);
698: PetscFree(nprocs);
700: /* pack messages to send back to local owners */
701: starts[0] = 0;
702: for (i=1; i<ng; i++) {
703: if (nownedsenders[i-1] > 1) starts[i] = starts[i-1] + nownedsenders[i-1];
704: else starts[i] = starts[i-1];
705: }
706: nsends2 = nrecvs;
707: PetscMalloc((nsends2+1)*sizeof(PetscInt),&nprocs); /* length of each message */
708: for (i=0; i<nrecvs; i++) {
709: nprocs[i] = 1;
710: for (j=0; j<len[i]; j++) {
711: node = recvs[2*i*nmax+2*j]-rstart;
712: if (nownedsenders[node] > 1) {
713: nprocs[i] += 2 + nownedsenders[node];
714: }
715: }
716: }
717: nt = 0; for (i=0; i<nsends2; i++) nt += nprocs[i];
718: PetscMalloc((nt+1)*sizeof(PetscInt),&sends2);
719: PetscMalloc((nsends2+1)*sizeof(PetscInt),&starts2);
720: starts2[0] = 0; for (i=1; i<nsends2; i++) starts2[i] = starts2[i-1] + nprocs[i-1];
721: /*
722: Each message is 1 + nprocs[i] long, and consists of
723: (0) the number of nodes being sent back
724: (1) the local node number,
725: (2) the number of processors sharing it,
726: (3) the processors sharing it
727: */
728: for (i=0; i<nsends2; i++) {
729: cnt = 1;
730: sends2[starts2[i]] = 0;
731: for (j=0; j<len[i]; j++) {
732: node = recvs[2*i*nmax+2*j]-rstart;
733: if (nownedsenders[node] > 1) {
734: sends2[starts2[i]]++;
735: sends2[starts2[i]+cnt++] = recvs[2*i*nmax+2*j+1];
736: sends2[starts2[i]+cnt++] = nownedsenders[node];
737: PetscMemcpy(&sends2[starts2[i]+cnt],&ownedsenders[starts[node]],nownedsenders[node]*sizeof(PetscInt));
738: cnt += nownedsenders[node];
739: }
740: }
741: }
743: /* receive the message lengths */
744: nrecvs2 = nsends;
745: PetscMalloc((nrecvs2+1)*sizeof(PetscInt),&lens2);
746: PetscMalloc((nrecvs2+1)*sizeof(PetscInt),&starts3);
747: PetscMalloc((nrecvs2+1)*sizeof(MPI_Request),&recv_waits);
748: for (i=0; i<nrecvs2; i++) {
749: MPI_Irecv(&lens2[i],1,MPIU_INT,dest[i],tag2,comm,recv_waits+i);
750: }
752: /* send the message lengths */
753: for (i=0; i<nsends2; i++) {
754: MPI_Send(&nprocs[i],1,MPIU_INT,source[i],tag2,comm);
755: }
757: /* wait on receives of lens */
758: if (nrecvs2) {
759: PetscMalloc(nrecvs2*sizeof(MPI_Status),&recv_statuses);
760: MPI_Waitall(nrecvs2,recv_waits,recv_statuses);
761: PetscFree(recv_statuses);
762: }
763: PetscFree(recv_waits);
765: starts3[0] = 0;
766: nt = 0;
767: for (i=0; i<nrecvs2-1; i++) {
768: starts3[i+1] = starts3[i] + lens2[i];
769: nt += lens2[i];
770: }
771: nt += lens2[nrecvs2-1];
773: PetscMalloc((nt+1)*sizeof(PetscInt),&recvs2);
774: PetscMalloc((nrecvs2+1)*sizeof(MPI_Request),&recv_waits);
775: for (i=0; i<nrecvs2; i++) {
776: MPI_Irecv(recvs2+starts3[i],lens2[i],MPIU_INT,dest[i],tag3,comm,recv_waits+i);
777: }
778:
779: /* send the messages */
780: PetscMalloc((nsends2+1)*sizeof(MPI_Request),&send_waits);
781: for (i=0; i<nsends2; i++) {
782: MPI_Isend(sends2+starts2[i],nprocs[i],MPIU_INT,source[i],tag3,comm,send_waits+i);
783: }
785: /* wait on receives */
786: if (nrecvs2) {
787: PetscMalloc(nrecvs2*sizeof(MPI_Status),&recv_statuses);
788: MPI_Waitall(nrecvs2,recv_waits,recv_statuses);
789: PetscFree(recv_statuses);
790: }
791: PetscFree(recv_waits);
792: PetscFree(nprocs);
794: if (debug) { /* ----------------------------------- */
795: cnt = 0;
796: for (i=0; i<nrecvs2; i++) {
797: nt = recvs2[cnt++];
798: for (j=0; j<nt; j++) {
799: PetscSynchronizedPrintf(comm,"[%d] local node %d number of subdomains %d: ",rank,recvs2[cnt],recvs2[cnt+1]);
800: for (k=0; k<recvs2[cnt+1]; k++) {
801: PetscSynchronizedPrintf(comm,"%d ",recvs2[cnt+2+k]);
802: }
803: cnt += 2 + recvs2[cnt+1];
804: PetscSynchronizedPrintf(comm,"\n");
805: }
806: }
807: PetscSynchronizedFlush(comm);
808: } /* ----------------------------------- */
810: /* count number subdomains for each local node */
811: PetscMalloc(size*sizeof(PetscInt),&nprocs);
812: PetscMemzero(nprocs,size*sizeof(PetscInt));
813: cnt = 0;
814: for (i=0; i<nrecvs2; i++) {
815: nt = recvs2[cnt++];
816: for (j=0; j<nt; j++) {
817: for (k=0; k<recvs2[cnt+1]; k++) {
818: nprocs[recvs2[cnt+2+k]]++;
819: }
820: cnt += 2 + recvs2[cnt+1];
821: }
822: }
823: nt = 0; for (i=0; i<size; i++) nt += (nprocs[i] > 0);
824: *nproc = nt;
825: PetscMalloc((nt+1)*sizeof(PetscInt),procs);
826: PetscMalloc((nt+1)*sizeof(PetscInt),numprocs);
827: PetscMalloc((nt+1)*sizeof(PetscInt*),indices);
828: PetscMalloc(size*sizeof(PetscInt),&bprocs);
829: cnt = 0;
830: for (i=0; i<size; i++) {
831: if (nprocs[i] > 0) {
832: bprocs[i] = cnt;
833: (*procs)[cnt] = i;
834: (*numprocs)[cnt] = nprocs[i];
835: PetscMalloc(nprocs[i]*sizeof(PetscInt),&(*indices)[cnt]);
836: cnt++;
837: }
838: }
840: /* make the list of subdomains for each nontrivial local node */
841: PetscMemzero(*numprocs,nt*sizeof(PetscInt));
842: cnt = 0;
843: for (i=0; i<nrecvs2; i++) {
844: nt = recvs2[cnt++];
845: for (j=0; j<nt; j++) {
846: for (k=0; k<recvs2[cnt+1]; k++) {
847: (*indices)[bprocs[recvs2[cnt+2+k]]][(*numprocs)[bprocs[recvs2[cnt+2+k]]]++] = recvs2[cnt];
848: }
849: cnt += 2 + recvs2[cnt+1];
850: }
851: }
852: PetscFree(bprocs);
853: PetscFree(recvs2);
855: /* sort the node indexing by their global numbers */
856: nt = *nproc;
857: for (i=0; i<nt; i++) {
858: PetscMalloc(((*numprocs)[i])*sizeof(PetscInt),&tmp);
859: for (j=0; j<(*numprocs)[i]; j++) {
860: tmp[j] = lindices[(*indices)[i][j]];
861: }
862: PetscSortIntWithArray((*numprocs)[i],tmp,(*indices)[i]);
863: PetscFree(tmp);
864: }
866: if (debug) { /* ----------------------------------- */
867: nt = *nproc;
868: for (i=0; i<nt; i++) {
869: PetscSynchronizedPrintf(comm,"[%d] subdomain %d number of indices %d: ",rank,(*procs)[i],(*numprocs)[i]);
870: for (j=0; j<(*numprocs)[i]; j++) {
871: PetscSynchronizedPrintf(comm,"%d ",(*indices)[i][j]);
872: }
873: PetscSynchronizedPrintf(comm,"\n");
874: }
875: PetscSynchronizedFlush(comm);
876: } /* ----------------------------------- */
878: /* wait on sends */
879: if (nsends2) {
880: PetscMalloc(nsends2*sizeof(MPI_Status),&send_status);
881: MPI_Waitall(nsends2,send_waits,send_status);
882: PetscFree(send_status);
883: }
885: PetscFree(starts3);
886: PetscFree(dest);
887: PetscFree(send_waits);
889: PetscFree(nownedsenders);
890: PetscFree(ownedsenders);
891: PetscFree(starts);
892: PetscFree(starts2);
893: PetscFree(lens2);
895: PetscFree(source);
896: PetscFree(len);
897: PetscFree(recvs);
898: PetscFree(nprocs);
899: PetscFree(sends2);
901: /* put the information about myself as the first entry in the list */
902: first_procs = (*procs)[0];
903: first_numprocs = (*numprocs)[0];
904: first_indices = (*indices)[0];
905: for (i=0; i<*nproc; i++) {
906: if ((*procs)[i] == rank) {
907: (*procs)[0] = (*procs)[i];
908: (*numprocs)[0] = (*numprocs)[i];
909: (*indices)[0] = (*indices)[i];
910: (*procs)[i] = first_procs;
911: (*numprocs)[i] = first_numprocs;
912: (*indices)[i] = first_indices;
913: break;
914: }
915: }
916: return(0);
917: }
921: /*@C
922: ISLocalToGlobalMappingRestoreInfo - Frees the memory allocated by ISLocalToGlobalMappingGetInfo()
924: Collective on ISLocalToGlobalMapping
926: Input Parameters:
927: . mapping - the mapping from local to global indexing
929: Output Parameter:
930: + nproc - number of processors that are connected to this one
931: . proc - neighboring processors
932: . numproc - number of indices for each processor
933: - indices - indices of local nodes shared with neighbor (sorted by global numbering)
935: Level: advanced
937: .seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreateIS(), ISLocalToGlobalMappingCreate(),
938: ISLocalToGlobalMappingGetInfo()
939: @*/
940: PetscErrorCode ISLocalToGlobalMappingRestoreInfo(ISLocalToGlobalMapping mapping,PetscInt *nproc,PetscInt *procs[],PetscInt *numprocs[],PetscInt **indices[])
941: {
943: PetscInt i;
946: PetscFree(*procs);
947: PetscFree(*numprocs);
948: if (*indices) {
949: PetscFree((*indices)[0]);
950: for (i=1; i<*nproc; i++) {
951: PetscFree((*indices)[i]);
952: }
953: PetscFree(*indices);
954: }
955: return(0);
956: }