Actual source code: bddcgraph.c

petsc-dev 2014-02-02
Report Typos and Errors
  1: #include <petsc-private/petscimpl.h>
  2: #include <bddcprivate.h>
  3: #include <bddcstructs.h>

  5: /* special marks: they cannot be enums, since special marks should in principle range from -4 to -max_int */
  6: #define NEUMANN_MARK -1
  7: #define DIRICHLET_MARK -2
  8: #define LOCAL_PERIODIC_MARK -3
  9: #define SPECIAL_MARK -4

 13: PetscErrorCode PCBDDCGraphASCIIView(PCBDDCGraph graph, PetscInt verbosity_level, PetscViewer viewer)
 14: {
 15:   PetscInt       i,j,tabs;
 16:   PetscInt*      queue_in_global_numbering;

 20:   PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
 21:   PetscViewerASCIIGetTab(viewer,&tabs);
 22:   PetscViewerASCIIPrintf(viewer,"--------------------------------------------------\n");
 23:   PetscViewerFlush(viewer);
 24:   PetscViewerASCIISynchronizedPrintf(viewer,"Local BDDC graph for subdomain %04d\n",PetscGlobalRank);
 25:   PetscViewerASCIISynchronizedPrintf(viewer,"Number of vertices %d\n",graph->nvtxs);
 26:   if (verbosity_level > 1) {
 27:     for (i=0;i<graph->nvtxs;i++) {
 28:       PetscViewerASCIISynchronizedPrintf(viewer,"%d:\n",i);
 29:       PetscViewerASCIISynchronizedPrintf(viewer,"   which_dof: %d\n",graph->which_dof[i]);
 30:       PetscViewerASCIISynchronizedPrintf(viewer,"   special_dof: %d\n",graph->special_dof[i]);
 31:       PetscViewerASCIISynchronizedPrintf(viewer,"   neighbours: %d\n",graph->count[i]);
 32:       PetscViewerASCIIUseTabs(viewer,PETSC_FALSE);
 33:       if (graph->count[i]) {
 34:         PetscViewerASCIISynchronizedPrintf(viewer,"     set of neighbours:");
 35:         for (j=0;j<graph->count[i];j++) {
 36:           PetscViewerASCIISynchronizedPrintf(viewer," %d",graph->neighbours_set[i][j]);
 37:         }
 38:         PetscViewerASCIISynchronizedPrintf(viewer,"\n");
 39:       }
 40:       PetscViewerASCIISetTab(viewer,tabs);
 41:       PetscViewerASCIIUseTabs(viewer,PETSC_TRUE);
 42:       if (graph->mirrors) {
 43:         PetscViewerASCIISynchronizedPrintf(viewer,"   mirrors: %d\n",graph->mirrors[i]);
 44:         if (graph->mirrors[i]) {
 45:           PetscViewerASCIIUseTabs(viewer,PETSC_FALSE);
 46:           PetscViewerASCIISynchronizedPrintf(viewer,"     set of mirrors:");
 47:           for (j=0;j<graph->mirrors[i];j++) {
 48:             PetscViewerASCIISynchronizedPrintf(viewer," %d",graph->mirrors_set[i][j]);
 49:           }
 50:           PetscViewerASCIISynchronizedPrintf(viewer,"\n");
 51:           PetscViewerASCIISetTab(viewer,tabs);
 52:           PetscViewerASCIIUseTabs(viewer,PETSC_TRUE);
 53:         }
 54:       }
 55:       if (verbosity_level > 2) {
 56:         if (graph->xadj && graph->adjncy) {
 57:           PetscViewerASCIISynchronizedPrintf(viewer,"   local adj list:");
 58:           PetscViewerASCIIUseTabs(viewer,PETSC_FALSE);
 59:           for (j=graph->xadj[i];j<graph->xadj[i+1];j++) {
 60:             PetscViewerASCIISynchronizedPrintf(viewer," %d",graph->adjncy[j]);
 61:           }
 62:           PetscViewerASCIISynchronizedPrintf(viewer,"\n");
 63:           PetscViewerASCIISetTab(viewer,tabs);
 64:           PetscViewerASCIIUseTabs(viewer,PETSC_TRUE);
 65:         }
 66:       }
 67:       PetscViewerASCIISynchronizedPrintf(viewer,"   interface subset id: %d\n",graph->subset[i]);
 68:       if (graph->subset[i] && graph->subset_ncc) {
 69:         PetscViewerASCIISynchronizedPrintf(viewer,"   ncc for subset: %d\n",graph->subset_ncc[graph->subset[i]-1]);
 70:       }
 71:     }
 72:   }
 73:   PetscViewerASCIISynchronizedPrintf(viewer,"Total number of connected components %d\n",graph->ncc);
 74:   PetscMalloc1(graph->cptr[graph->ncc],&queue_in_global_numbering);
 75:   ISLocalToGlobalMappingApply(graph->l2gmap,graph->cptr[graph->ncc],graph->queue,queue_in_global_numbering);
 76:   for (i=0;i<graph->ncc;i++) {
 77:     PetscViewerASCIISynchronizedPrintf(viewer,"  %d (neighs:",i);
 78:     PetscViewerASCIIUseTabs(viewer,PETSC_FALSE);
 79:     PetscInt node_num=graph->queue[graph->cptr[i]];
 80:     for (j=0;j<graph->count[node_num];j++) {
 81:       PetscViewerASCIISynchronizedPrintf(viewer," %d",graph->neighbours_set[node_num][j]);
 82:     }
 83:     PetscViewerASCIISynchronizedPrintf(viewer,"):");
 84:     for (j=graph->cptr[i];j<graph->cptr[i+1];j++) {
 85:       PetscViewerASCIISynchronizedPrintf(viewer," %d (%d)",graph->queue[j],queue_in_global_numbering[j]);
 86:     }
 87:     PetscViewerASCIISynchronizedPrintf(viewer,"\n");
 88:     PetscViewerASCIISetTab(viewer,tabs);
 89:     PetscViewerASCIIUseTabs(viewer,PETSC_TRUE);
 90:   }
 91:   PetscFree(queue_in_global_numbering);
 92:   if (graph->custom_minimal_size > 1 && verbosity_level > 1) {
 93:     PetscViewerASCIISynchronizedPrintf(viewer,"Custom minimal size %d\n",graph->custom_minimal_size);
 94:   }
 95:   PetscViewerFlush(viewer);
 96:   return(0);
 97: }

101: PetscErrorCode PCBDDCGraphGetCandidatesIS(PCBDDCGraph graph, PetscBool use_faces, PetscBool use_edges, PetscBool use_vertices, PetscInt *n_faces, IS *FacesIS[], PetscInt *n_edges, IS *EdgesIS[], IS *VerticesIS)
102: {
103:   IS             *ISForFaces,*ISForEdges,ISForVertices;
104:   PetscInt       i,j,nfc,nec,nvc,*idx;
105:   PetscBool      twodim_flag;

109:   /* loop on ccs to evalute number of faces, edges and vertices */
110:   nfc = 0;
111:   nec = 0;
112:   nvc = 0;
113:   twodim_flag = PETSC_FALSE;
114:   for (i=0;i<graph->ncc;i++) {
115:     if (graph->cptr[i+1]-graph->cptr[i] > graph->custom_minimal_size) {
116:       if (graph->count[graph->queue[graph->cptr[i]]] == 1 && graph->special_dof[graph->queue[graph->cptr[i]]] != NEUMANN_MARK) {
117:         nfc++;
118:       } else { /* note that nec will be zero in 2d */
119:         nec++;
120:       }
121:     } else {
122:       nvc += graph->cptr[i+1]-graph->cptr[i];
123:     }
124:   }
125:   if (!nec) { /* we are in a 2d case -> no faces, only edges */
126:     nec = nfc;
127:     nfc = 0;
128:     twodim_flag = PETSC_TRUE;
129:   }
130:   /* allocate IS arrays for faces, edges. Vertices need a single index set. */
131:   ISForFaces = 0;
132:   ISForEdges = 0;
133:   ISForVertices = 0;
134:   if (use_faces && nfc) {
135:     PetscMalloc1(nfc,&ISForFaces);
136:   }
137:   if (use_edges && nec) {
138:     PetscMalloc1(nec,&ISForEdges);
139:   }
140:   if (use_vertices && nvc) {
141:     PetscMalloc1(nvc,&idx);
142:   }
143:   /* loop on ccs to compute index sets for faces and edges */
144:   nfc = 0;
145:   nec = 0;
146:   for (i=0;i<graph->ncc;i++) {
147:     if (graph->cptr[i+1]-graph->cptr[i] > graph->custom_minimal_size) {
148:       if (graph->count[graph->queue[graph->cptr[i]]] == 1 && graph->special_dof[graph->queue[graph->cptr[i]]] != NEUMANN_MARK) {
149:         if (twodim_flag) {
150:           if (use_edges) {
151:             ISCreateGeneral(PETSC_COMM_SELF,graph->cptr[i+1]-graph->cptr[i],&graph->queue[graph->cptr[i]],PETSC_COPY_VALUES,&ISForEdges[nec]);
152:             nec++;
153:           }
154:         } else {
155:           if (use_faces) {
156:             ISCreateGeneral(PETSC_COMM_SELF,graph->cptr[i+1]-graph->cptr[i],&graph->queue[graph->cptr[i]],PETSC_COPY_VALUES,&ISForFaces[nfc]);
157:             nfc++;
158:           }
159:         }
160:       } else {
161:         if (use_edges) {
162:           ISCreateGeneral(PETSC_COMM_SELF,graph->cptr[i+1]-graph->cptr[i],&graph->queue[graph->cptr[i]],PETSC_COPY_VALUES,&ISForEdges[nec]);
163:           nec++;
164:         }
165:       }
166:     }
167:   }
168:   /* index set for vertices */
169:   if (use_vertices && nvc) {
170:     nvc = 0;
171:     for (i=0;i<graph->ncc;i++) {
172:       if (graph->cptr[i+1]-graph->cptr[i] <= graph->custom_minimal_size) {
173:         for (j=graph->cptr[i];j<graph->cptr[i+1];j++) {
174:           idx[nvc]=graph->queue[j];
175:           nvc++;
176:         }
177:       }
178:     }
179:     /* sort vertex set (by local ordering) */
180:     PetscSortInt(nvc,idx);
181:     ISCreateGeneral(PETSC_COMM_SELF,nvc,idx,PETSC_OWN_POINTER,&ISForVertices);
182:   }
183:   /* get back info */
184:   *n_faces = nfc;
185:   *FacesIS = ISForFaces;
186:   *n_edges = nec;
187:   *EdgesIS = ISForEdges;
188:   *VerticesIS = ISForVertices;
189:   return(0);
190: }

194: PetscErrorCode PCBDDCGraphComputeConnectedComponents(PCBDDCGraph graph)
195: {
196:   PetscInt    adapt_interface,adapt_interface_reduced;
197:   MPI_Comm    interface_comm;
198:   MPI_Request *send_requests;
199:   MPI_Request *recv_requests;
200:   PetscInt    *aux_new_xadj,*new_xadj,*new_adjncy,**temp_buffer;
201:   PetscInt    i,j,k,s,sum_requests,buffer_size,size_of_recv,temp_buffer_size;
202:   PetscMPIInt rank,neigh,tag,mpi_buffer_size;
203:   PetscInt    *cum_recv_counts,*subset_to_nodes_indices,*recv_buffer_subset,*nodes_to_temp_buffer_indices;
204:   PetscInt    *send_buffer,*recv_buffer,*queue_in_global_numbering,*sizes_of_sends,*add_to_subset;
205:   PetscInt    start_of_recv,start_of_send,size_of_send,global_subset_counter,ins_val;
206:   PetscBool   *subset_cc_adapt,same_set;

210:   /* compute connected components locally */
211:   PetscObjectGetComm((PetscObject)(graph->l2gmap),&interface_comm);
212:   PCBDDCGraphComputeConnectedComponentsLocal(graph);
213:   /* check consistency of connected components among neighbouring subdomains -> it adapt them in case it is needed */
214:   adapt_interface = 0;
215:   adapt_interface_reduced = 0;
216:   for (i=0;i<graph->n_subsets;i++) {
217:     /* We are not sure that on a given subset of the local interface,
218:        with two connected components, the latters be the same among sharing subdomains */
219:     if (graph->subset_ncc[i] > 1) {
220:       adapt_interface=1;
221:       break;
222:     }
223:   }
224:   MPI_Allreduce(&adapt_interface,&adapt_interface_reduced,1,MPIU_INT,MPI_LOR,interface_comm);

226:   if (graph->n_subsets && adapt_interface_reduced) {
227:     /* Retrict adjacency graph using information from previously computed connected components */
228:     PetscMalloc1(graph->nvtxs,&aux_new_xadj);
229:     for (i=0;i<graph->nvtxs;i++) {
230:       aux_new_xadj[i]=1;
231:     }
232:     for (i=0;i<graph->ncc;i++) {
233:       k = graph->cptr[i+1]-graph->cptr[i];
234:       for (j=0;j<k;j++) {
235:         aux_new_xadj[graph->queue[graph->cptr[i]+j]]=k;
236:       }
237:     }
238:     j = 0;
239:     for (i=0;i<graph->nvtxs;i++) {
240:       j += aux_new_xadj[i];
241:     }
242:     PetscMalloc1((graph->nvtxs+1),&new_xadj);
243:     PetscMalloc1(j,&new_adjncy);
244:     new_xadj[0]=0;
245:     for (i=0;i<graph->nvtxs;i++) {
246:       new_xadj[i+1]=new_xadj[i]+aux_new_xadj[i];
247:       if (aux_new_xadj[i]==1) {
248:         new_adjncy[new_xadj[i]]=i;
249:       }
250:     }
251:     PetscFree(aux_new_xadj);
252:     for (i=0;i<graph->ncc;i++) {
253:       k = graph->cptr[i+1]-graph->cptr[i];
254:       for (j=0;j<k;j++) {
255:         PetscMemcpy(&new_adjncy[new_xadj[graph->queue[graph->cptr[i]+j]]],&graph->queue[graph->cptr[i]],k*sizeof(PetscInt));
256:       }
257:     }
258:     /* set new CSR into graph */
259:     PetscFree(graph->xadj);
260:     PetscFree(graph->adjncy);
261:     graph->xadj = new_xadj;
262:     graph->adjncy = new_adjncy;
263:     /* allocate some space */
264:     MPI_Comm_rank(interface_comm,&rank);
265:     PetscMalloc1((graph->n_subsets+1),&cum_recv_counts);
266:     PetscMemzero(cum_recv_counts,(graph->n_subsets+1)*sizeof(*cum_recv_counts));
267:     PetscMalloc1(graph->n_subsets,&subset_to_nodes_indices);
268:     /* first count how many neighbours per connected component I will receive from */
269:     cum_recv_counts[0]=0;
270:     for (i=1;i<graph->n_subsets+1;i++) {
271:       j = 0;
272:       while (graph->subset[j] != i) {
273:         j++;
274:       }
275:       subset_to_nodes_indices[i-1]=j;
276:       /* We don't want sends/recvs_to/from_self -> here I don't count myself  */
277:       cum_recv_counts[i]=cum_recv_counts[i-1]+graph->count[j];
278:     }
279:     PetscMalloc1(2*cum_recv_counts[graph->n_subsets],&recv_buffer_subset);
280:     PetscMalloc1(cum_recv_counts[graph->n_subsets],&send_requests);
281:     PetscMalloc1(cum_recv_counts[graph->n_subsets],&recv_requests);
282:     for (i=0;i<cum_recv_counts[graph->n_subsets];i++) {
283:       send_requests[i]=MPI_REQUEST_NULL;
284:       recv_requests[i]=MPI_REQUEST_NULL;
285:     }
286:     /* exchange with my neighbours the number of my connected components on the shared interface */
287:     sum_requests = 0;
288:     for (i=0;i<graph->n_subsets;i++) {
289:       j = subset_to_nodes_indices[i];
290:       for (k=0;k<graph->count[j];k++) {
291:         PetscMPIIntCast(graph->neighbours_set[j][k],&neigh);
292:         PetscMPIIntCast((PetscInt)(rank+1)*graph->count[j],&tag);
293:         MPI_Isend(&graph->subset_ncc[i],1,MPIU_INT,neigh,tag,interface_comm,&send_requests[sum_requests]);
294:         PetscMPIIntCast((PetscInt)(neigh+1)*graph->count[j],&tag);
295:         MPI_Irecv(&recv_buffer_subset[sum_requests],1,MPIU_INT,neigh,tag,interface_comm,&recv_requests[sum_requests]);
296:         sum_requests++;
297:       }
298:     }
299:     MPI_Waitall(sum_requests,recv_requests,MPI_STATUSES_IGNORE);
300:     MPI_Waitall(sum_requests,send_requests,MPI_STATUSES_IGNORE);
301:     /* determine the connected component I need to adapt */
302:     PetscMalloc1(graph->n_subsets,&subset_cc_adapt);
303:     PetscMemzero(subset_cc_adapt,graph->n_subsets*sizeof(*subset_cc_adapt));
304:     for (i=0;i<graph->n_subsets;i++) {
305:       for (j=cum_recv_counts[i];j<cum_recv_counts[i+1];j++){
306:         /* The first condition is natural (someone has a different number of ccs than me), the second one is just to be safe */
307:         if ( graph->subset_ncc[i] != recv_buffer_subset[j] || graph->subset_ncc[i] > 1 ) {
308:           subset_cc_adapt[i] = PETSC_TRUE;
309:           break;
310:         }
311:       }
312:     }
313:     buffer_size = 0;
314:     for (i=0;i<graph->n_subsets;i++) {
315:       if (subset_cc_adapt[i]) {
316:         for (j=i;j<graph->ncc;j++) {
317:           if (graph->subset[graph->queue[graph->cptr[j]]] == i+1) { /* WARNING -> subset values goes from 1 to graph->n_subsets included */
318:             buffer_size += 1 + graph->cptr[j+1]-graph->cptr[j];
319:           }
320:         }
321:       }
322:     }
323:     PetscMalloc1(buffer_size,&send_buffer);
324:     /* now get from neighbours their ccs (in global numbering) and adapt them (in case it is needed) */
325:     PetscMalloc1(graph->cptr[graph->ncc],&queue_in_global_numbering);
326:     ISLocalToGlobalMappingApply(graph->l2gmap,graph->cptr[graph->ncc],graph->queue,queue_in_global_numbering);
327:     /* determine how much data to send (size of each queue plus the global indices) and communicate it to neighbours */
328:     PetscMalloc1(graph->n_subsets,&sizes_of_sends);
329:     PetscMemzero(sizes_of_sends,graph->n_subsets*sizeof(*sizes_of_sends));
330:     sum_requests = 0;
331:     start_of_send = 0;
332:     start_of_recv = cum_recv_counts[graph->n_subsets];
333:     for (i=0;i<graph->n_subsets;i++) {
334:       if (subset_cc_adapt[i]) {
335:         size_of_send = 0;
336:         for (j=i;j<graph->ncc;j++) {
337:           if (graph->subset[graph->queue[graph->cptr[j]]] == i+1) { /* WARNING -> subset values goes from 1 to graph->n_subsets included */
338:             send_buffer[start_of_send+size_of_send]=graph->cptr[j+1]-graph->cptr[j];
339:             size_of_send += 1;
340:             PetscMemcpy(&send_buffer[start_of_send+size_of_send],
341:                                &queue_in_global_numbering[graph->cptr[j]],
342:                                (graph->cptr[j+1]-graph->cptr[j])*sizeof(*send_buffer));
343:             size_of_send = size_of_send+graph->cptr[j+1]-graph->cptr[j];
344:           }
345:         }
346:         j = subset_to_nodes_indices[i];
347:         sizes_of_sends[i] = size_of_send;
348:         for (k=0;k<graph->count[j];k++) {
349:           PetscMPIIntCast(graph->neighbours_set[j][k],&neigh);
350:           PetscMPIIntCast((PetscInt)(rank+1)*graph->count[j],&tag);
351:           MPI_Isend(&sizes_of_sends[i],1,MPIU_INT,neigh,tag,interface_comm,&send_requests[sum_requests]);
352:           PetscMPIIntCast((PetscInt)(neigh+1)*graph->count[j],&tag);
353:           MPI_Irecv(&recv_buffer_subset[sum_requests+start_of_recv],1,MPIU_INT,neigh,tag,interface_comm,&recv_requests[sum_requests]);
354:           sum_requests++;
355:         }
356:         start_of_send += size_of_send;
357:       }
358:     }
359:     MPI_Waitall(sum_requests,send_requests,MPI_STATUSES_IGNORE);
360:     MPI_Waitall(sum_requests,recv_requests,MPI_STATUSES_IGNORE);
361:     buffer_size = 0;
362:     for (k=0;k<sum_requests;k++) {
363:       buffer_size += recv_buffer_subset[start_of_recv+k];
364:     }
365:     PetscMalloc1(buffer_size,&recv_buffer);
366:     /* now exchange the data */
367:     start_of_recv = 0;
368:     start_of_send = 0;
369:     sum_requests = 0;
370:     for (i=0;i<graph->n_subsets;i++) {
371:       if (subset_cc_adapt[i]) {
372:         size_of_send = sizes_of_sends[i];
373:         j = subset_to_nodes_indices[i];
374:         for (k=0;k<graph->count[j];k++) {
375:           PetscMPIIntCast(graph->neighbours_set[j][k],&neigh);
376:           PetscMPIIntCast((PetscInt)(rank+1)*graph->count[j],&tag);
377:           PetscMPIIntCast(size_of_send,&mpi_buffer_size);
378:           MPI_Isend(&send_buffer[start_of_send],mpi_buffer_size,MPIU_INT,neigh,tag,interface_comm,&send_requests[sum_requests]);
379:           size_of_recv = recv_buffer_subset[cum_recv_counts[graph->n_subsets]+sum_requests];
380:           PetscMPIIntCast((PetscInt)(neigh+1)*graph->count[j],&tag);
381:           PetscMPIIntCast(size_of_recv,&mpi_buffer_size);
382:           MPI_Irecv(&recv_buffer[start_of_recv],mpi_buffer_size,MPIU_INT,neigh,tag,interface_comm,&recv_requests[sum_requests]);
383:           start_of_recv += size_of_recv;
384:           sum_requests++;
385:         }
386:         start_of_send += size_of_send;
387:       }
388:     }
389:     MPI_Waitall(sum_requests,recv_requests,MPI_STATUSES_IGNORE);
390:     MPI_Waitall(sum_requests,send_requests,MPI_STATUSES_IGNORE);
391:     for (j=0;j<buffer_size;) {
392:        ISGlobalToLocalMappingApply(graph->l2gmap,IS_GTOLM_MASK,recv_buffer[j],&recv_buffer[j+1],&recv_buffer[j],&recv_buffer[j+1]);
393:        /* we need to adapt the output of GlobalToLocal mapping if there are mirrored nodes */
394:        if (graph->mirrors) {
395:          PetscBool mirrored_found=PETSC_FALSE;
396:          for (k=0;k<recv_buffer[j];k++) {
397:            if (graph->mirrors[recv_buffer[j+k+1]]) {
398:              mirrored_found=PETSC_TRUE;
399:              recv_buffer[j+k+1]=graph->mirrors_set[recv_buffer[j+k+1]][0];
400:            }
401:          }
402:          if (mirrored_found) {
403:            PetscSortInt(recv_buffer[j],&recv_buffer[j+1]);
404:            k=0;
405:            while (k<recv_buffer[j]) {
406:              for (s=1;s<graph->mirrors[recv_buffer[j+1+k]];s++) {
407:                recv_buffer[j+1+k+s] = graph->mirrors_set[recv_buffer[j+1+k]][s];
408:              }
409:              k+=graph->mirrors[recv_buffer[j+1+k]]+s;
410:            }
411:          }
412:        }
413:        k = recv_buffer[j]+1;
414:        j += k;
415:     }
416:     sum_requests = cum_recv_counts[graph->n_subsets];
417:     start_of_recv = 0;
418:     PetscMalloc1(graph->nvtxs,&nodes_to_temp_buffer_indices);
419:     global_subset_counter = 0;
420:     for (i=0;i<graph->n_subsets;i++) {
421:       if (subset_cc_adapt[i]) {
422:         temp_buffer_size = 0;
423:         /* find nodes on the shared interface we need to adapt */
424:         for (j=0;j<graph->nvtxs;j++) {
425:           if (graph->subset[j]==i+1) {
426:             nodes_to_temp_buffer_indices[j] = temp_buffer_size;
427:             temp_buffer_size++;
428:           } else {
429:             nodes_to_temp_buffer_indices[j] = -1;
430:           }
431:         }
432:         /* allocate some temporary space */
433:         PetscMalloc1(temp_buffer_size,&temp_buffer);
434:         PetscMalloc1(temp_buffer_size*(cum_recv_counts[i+1]-cum_recv_counts[i]),&temp_buffer[0]);
435:         PetscMemzero(temp_buffer[0],temp_buffer_size*(cum_recv_counts[i+1]-cum_recv_counts[i])*sizeof(PetscInt));
436:         for (j=1;j<temp_buffer_size;j++) {
437:           temp_buffer[j] = temp_buffer[j-1]+cum_recv_counts[i+1]-cum_recv_counts[i];
438:         }
439:         /* analyze contributions from neighbouring subdomains for i-th conn comp
440:            temp buffer structure:
441:            supposing part of the interface has dimension 5 (for example with global dofs 0,1,2,3,4)
442:            3 neighs procs with structured connected components:
443:              neigh 0: [0 1 4], [2 3];  (2 connected components)
444:              neigh 1: [0 1], [2 3 4];  (2 connected components)
445:              neigh 2: [0 4], [1], [2 3]; (3 connected components)
446:            tempbuffer (row-oriented) will be filled as:
447:              [ 0, 0, 0;
448:                0, 0, 1;
449:                1, 1, 2;
450:                1, 1, 2;
451:                0, 1, 0; ];
452:            This way we can simply find intersections of ccs among neighs.
453:            For the example above, the graph->subset array will be modified to reproduce the following 4 connected components [0], [1], [2 3], [4];
454:                                                                                                                                    */
455:         for (j=0;j<cum_recv_counts[i+1]-cum_recv_counts[i];j++) {
456:           ins_val = 0;
457:           size_of_recv = recv_buffer_subset[sum_requests];  /* total size of recv from neighs */
458:           for (buffer_size=0;buffer_size<size_of_recv;) {  /* loop until all data from neighs has been taken into account */
459:             for (k=1;k<recv_buffer[buffer_size+start_of_recv]+1;k++) { /* filling properly temp_buffer using data from a single recv */
460:               temp_buffer[nodes_to_temp_buffer_indices[recv_buffer[start_of_recv+buffer_size+k]]][j] = ins_val;
461:             }
462:             buffer_size += k;
463:             ins_val++;
464:           }
465:           start_of_recv += size_of_recv;
466:           sum_requests++;
467:         }
468:         PetscMalloc1(temp_buffer_size,&add_to_subset);
469:         PetscMemzero(add_to_subset,temp_buffer_size*sizeof(*add_to_subset));
470:         for (j=0;j<temp_buffer_size;j++) {
471:           if (!add_to_subset[j]) { /* found a new cc  */
472:             global_subset_counter++;
473:             add_to_subset[j] = global_subset_counter;
474:             for (k=j+1;k<temp_buffer_size;k++) { /* check for other nodes in new cc */
475:               same_set = PETSC_TRUE;
476:               for (s=0;s<cum_recv_counts[i+1]-cum_recv_counts[i];s++) {
477:                 if (temp_buffer[j][s]!=temp_buffer[k][s]) {
478:                   same_set = PETSC_FALSE;
479:                   break;
480:                 }
481:               }
482:               if (same_set) {
483:                 add_to_subset[k] = global_subset_counter;
484:               }
485:             }
486:           }
487:         }
488:         /* insert new data in subset array */
489:         temp_buffer_size = 0;
490:         for (j=0;j<graph->nvtxs;j++) {
491:           if (graph->subset[j]==i+1) {
492:             graph->subset[j] = graph->n_subsets+add_to_subset[temp_buffer_size];
493:             temp_buffer_size++;
494:           }
495:         }
496:         PetscFree(temp_buffer[0]);
497:         PetscFree(temp_buffer);
498:         PetscFree(add_to_subset);
499:       }
500:     }
501:     PetscFree(nodes_to_temp_buffer_indices);
502:     PetscFree(sizes_of_sends);
503:     PetscFree(send_requests);
504:     PetscFree(recv_requests);
505:     PetscFree(recv_buffer);
506:     PetscFree(recv_buffer_subset);
507:     PetscFree(send_buffer);
508:     PetscFree(cum_recv_counts);
509:     PetscFree(subset_to_nodes_indices);
510:     PetscFree(subset_cc_adapt);
511:     /* We are ready to find for connected components consistent among neighbouring subdomains */
512:     if (global_subset_counter) {
513:       PetscBTMemzero(graph->nvtxs,graph->touched);
514:       global_subset_counter = 0;
515:       for (i=0;i<graph->nvtxs;i++) {
516:         if (graph->subset[i] && !PetscBTLookup(graph->touched,i)) {
517:           global_subset_counter++;
518:           for (j=i+1;j<graph->nvtxs;j++) {
519:             if (!PetscBTLookup(graph->touched,j) && graph->subset[j]==graph->subset[i]) {
520:               graph->subset[j] = global_subset_counter;
521:               PetscBTSet(graph->touched,j);
522:             }
523:           }
524:           graph->subset[i] = global_subset_counter;
525:           PetscBTSet(graph->touched,i);
526:         }
527:       }
528:       /* refine connected components locally */
529:       PCBDDCGraphComputeConnectedComponentsLocal(graph);
530:     }
531:     PetscFree(queue_in_global_numbering);
532:   }
533:   return(0);
534: }

536: /* The following code has been adapted from function IsConnectedSubdomain contained
537:    in source file contig.c of METIS library (version 5.0.1)
538:    It finds connected components for each subset  */
541: PetscErrorCode PCBDDCGraphComputeConnectedComponentsLocal(PCBDDCGraph graph)
542: {
543:   PetscInt       i,j,k,first,last,nleft,ncc,pid,cum_queue,n,ncc_pid;
544:   PetscInt       *queue_global;

548:   /* quiet return if no csr info is available */
549:   if (!graph->xadj || !graph->adjncy) {
550:     return(0);
551:   }

553:   /* reset any previous search of connected components */
554:   PetscBTMemzero(graph->nvtxs,graph->touched);
555:   graph->n_subsets = 0;
556:   for (i=0;i<graph->nvtxs;i++) {
557:     if (graph->special_dof[i] == DIRICHLET_MARK || !graph->count[i]) {
558:       PetscBTSet(graph->touched,i);
559:       graph->subset[i] = 0;
560:     }
561:     graph->n_subsets = PetscMax(graph->n_subsets,graph->subset[i]);
562:   }
563:   PetscFree(graph->subset_ncc);
564:   PetscMalloc1(graph->n_subsets,&graph->subset_ncc);
565:   PetscMemzero(graph->subset_ncc,graph->n_subsets*sizeof(*graph->subset_ncc));
566:   PetscMemzero(graph->cptr,(graph->nvtxs+1)*sizeof(*graph->cptr));
567:   PetscMemzero(graph->queue,graph->nvtxs*sizeof(*graph->queue));

569:   /* begin search for connected components */
570:   cum_queue = 0;
571:   ncc = 0;
572:   for (n=0;n<graph->n_subsets;n++) {
573:     pid = n+1;  /* partition labeled by 0 is discarded */
574:     nleft = 0;
575:     for (i=0;i<graph->nvtxs;i++) {
576:       if (graph->subset[i] == pid) {
577:         nleft++;
578:       }
579:     }
580:     for (i=0; i<graph->nvtxs; i++) {
581:       if (graph->subset[i] == pid) {
582:         break;
583:       }
584:     }
585:     PetscBTSet(graph->touched,i);
586:     graph->queue[cum_queue] = i;
587:     first = 0;
588:     last = 1;
589:     graph->cptr[ncc] = cum_queue;
590:     ncc_pid = 0;
591:     while (first != nleft) {
592:       if (first == last) {
593:         graph->cptr[++ncc] = first+cum_queue;
594:         ncc_pid++;
595:         for (i=0; i<graph->nvtxs; i++) { /* TODO-> use a while! */
596:           if (graph->subset[i] == pid && !PetscBTLookup(graph->touched,i)) {
597:             break;
598:           }
599:         }
600:         graph->queue[cum_queue+last] = i;
601:         last++;
602:         PetscBTSet(graph->touched,i);
603:       }
604:       i = graph->queue[cum_queue+first];
605:       first++;
606:       for (j=graph->xadj[i];j<graph->xadj[i+1];j++) {
607:         k = graph->adjncy[j];
608:         if (graph->subset[k] == pid && !PetscBTLookup(graph->touched,k)) {
609:           graph->queue[cum_queue+last] = k;
610:           last++;
611:           PetscBTSet(graph->touched,k);
612:         }
613:       }
614:     }
615:     graph->cptr[++ncc] = first+cum_queue;
616:     ncc_pid++;
617:     cum_queue = graph->cptr[ncc];
618:     graph->subset_ncc[n] = ncc_pid;
619:   }
620:   graph->ncc = ncc;
621:   /* For consistency among neighbours, I need to sort (by global ordering) each connected component */
622:   PetscMalloc1(graph->cptr[graph->ncc],&queue_global);
623:   ISLocalToGlobalMappingApply(graph->l2gmap,graph->cptr[graph->ncc],graph->queue,queue_global);
624:   for (i=0;i<graph->ncc;i++) {
625:     PetscSortIntWithArray(graph->cptr[i+1]-graph->cptr[i],&queue_global[graph->cptr[i]],&graph->queue[graph->cptr[i]]);
626:   }
627:   PetscFree(queue_global);
628:   return(0);
629: }

633: PetscErrorCode PCBDDCGraphSetUp(PCBDDCGraph graph, PetscInt custom_minimal_size, IS neumann_is, IS dirichlet_is, PetscInt n_ISForDofs, IS ISForDofs[], IS custom_primal_vertices)
634: {
635:   VecScatter     scatter_ctx;
636:   Vec            local_vec,local_vec2,global_vec;
637:   IS             to,from;
638:   MPI_Comm       comm;
639:   PetscScalar    *array,*array2;
640:   const PetscInt *is_indices;
641:   PetscInt       n_neigh,*neigh,*n_shared,**shared;
642:   PetscInt       i,j,k,s,total_counts,nodes_touched,is_size;
644:   PetscBool      same_set,mirrors_found;

647:   PetscObjectGetComm((PetscObject)(graph->l2gmap),&comm);
648:   /* custom_minimal_size */
649:   graph->custom_minimal_size = PetscMax(graph->custom_minimal_size,custom_minimal_size);
650:   /* get info l2gmap and allocate work vectors  */
651:   ISLocalToGlobalMappingGetInfo(graph->l2gmap,&n_neigh,&neigh,&n_shared,&shared);
652:   ISLocalToGlobalMappingGetIndices(graph->l2gmap,&is_indices);
653:   j = 0;
654:   for (i=0;i<graph->nvtxs;i++) {
655:     j = PetscMax(j,is_indices[i]);
656:   }
657:   MPI_Allreduce(&j,&i,1,MPIU_INT,MPI_MAX,comm);
658:   i++;
659:   VecCreate(PETSC_COMM_SELF,&local_vec);
660:   VecSetSizes(local_vec,PETSC_DECIDE,graph->nvtxs);
661:   VecSetType(local_vec,VECSTANDARD);
662:   VecDuplicate(local_vec,&local_vec2);
663:   VecCreate(comm,&global_vec);
664:   VecSetSizes(global_vec,PETSC_DECIDE,i);
665:   VecSetType(global_vec,VECSTANDARD);
666:   ISCreateStride(PETSC_COMM_SELF,graph->nvtxs,0,1,&to);
667:   ISLocalToGlobalMappingApplyIS(graph->l2gmap,to,&from);
668:   VecScatterCreate(global_vec,from,local_vec,to,&scatter_ctx);

670:   /* get local periodic nodes */
671:   mirrors_found = PETSC_FALSE;
672:   if (graph->nvtxs && n_neigh) {
673:     for (i=0; i<n_shared[0]; i++) graph->count[shared[0][i]] += 1;
674:     for (i=0; i<n_shared[0]; i++) {
675:       if (graph->count[shared[0][i]] > 1) {
676:         mirrors_found = PETSC_TRUE;
677:         break;
678:       }
679:     }
680:   }
681:   /* compute local mirrors (if any) */
682:   if (mirrors_found) {
683:     PetscInt *local_indices,*global_indices;
684:     /* get arrays of local and global indices */
685:     PetscMalloc1(graph->nvtxs,&local_indices);
686:     ISGetIndices(to,(const PetscInt**)&is_indices);
687:     PetscMemcpy(local_indices,is_indices,graph->nvtxs*sizeof(PetscInt));
688:     ISRestoreIndices(to,(const PetscInt**)&is_indices);
689:     PetscMalloc1(graph->nvtxs,&global_indices);
690:     ISGetIndices(from,(const PetscInt**)&is_indices);
691:     PetscMemcpy(global_indices,is_indices,graph->nvtxs*sizeof(PetscInt));
692:     ISRestoreIndices(from,(const PetscInt**)&is_indices);
693:     /* allocate space for mirrors */
694:     PetscMalloc2(graph->nvtxs,&graph->mirrors,graph->nvtxs,&graph->mirrors_set);
695:     PetscMemzero(graph->mirrors,graph->nvtxs*sizeof(PetscInt));
696:     graph->mirrors_set[0] = 0;

698:     k=0;
699:     for (i=0;i<n_shared[0];i++) {
700:       j=shared[0][i];
701:       if (graph->count[j] > 1) {
702:         graph->mirrors[j]++;
703:         k++;
704:       }
705:     }
706:     /* allocate space for set of mirrors */
707:     PetscMalloc1(k,&graph->mirrors_set[0]);
708:     for (i=1;i<graph->nvtxs;i++)
709:       graph->mirrors_set[i]=graph->mirrors_set[i-1]+graph->mirrors[i-1];

711:     /* fill arrays */
712:     PetscMemzero(graph->mirrors,graph->nvtxs*sizeof(PetscInt));
713:     for (j=0;j<n_shared[0];j++) {
714:       i=shared[0][j];
715:       if (graph->count[i] > 1)
716:         graph->mirrors_set[i][graph->mirrors[i]++]=global_indices[i];
717:     }
718:     PetscSortIntWithArray(graph->nvtxs,global_indices,local_indices);
719:     for (i=0;i<graph->nvtxs;i++) {
720:       if (graph->mirrors[i] > 0) {
721:         PetscFindInt(graph->mirrors_set[i][0],graph->nvtxs,global_indices,&k);
722:         j = global_indices[k];
723:         while ( k > 0 && global_indices[k-1] == j) k--;
724:         for (j=0;j<graph->mirrors[i];j++) {
725:           graph->mirrors_set[i][j]=local_indices[k+j];
726:         }
727:         PetscSortInt(graph->mirrors[i],graph->mirrors_set[i]);
728:       }
729:     }
730:     PetscFree(local_indices);
731:     PetscFree(global_indices);
732:   }
733:   PetscMemzero(graph->count,graph->nvtxs*sizeof(*graph->count));
734:   ISDestroy(&to);
735:   ISDestroy(&from);

737:   /* Count total number of neigh per node */
738:   k=0;
739:   for (i=1;i<n_neigh;i++) {
740:     k += n_shared[i];
741:     for (j=0;j<n_shared[i];j++) {
742:       graph->count[shared[i][j]] += 1;
743:     }
744:   }
745:   /* Allocate space for storing the set of neighbours for each node */
746:   if (graph->nvtxs) {
747:     PetscMalloc1(k,&graph->neighbours_set[0]);
748:   }
749:   for (i=1;i<graph->nvtxs;i++) { /* dont count myself */
750:     graph->neighbours_set[i]=graph->neighbours_set[i-1]+graph->count[i-1];
751:   }
752:   /* Get information for sharing subdomains */
753:   PetscMemzero(graph->count,graph->nvtxs*sizeof(*graph->count));
754:   for (i=1;i<n_neigh;i++) { /* dont count myself */
755:     s = n_shared[i];
756:     for (j=0;j<s;j++) {
757:       k = shared[i][j];
758:       graph->neighbours_set[k][graph->count[k]] = neigh[i];
759:       graph->count[k] += 1;
760:     }
761:   }
762:   /* sort set of sharing subdomains */
763:   for (i=0;i<graph->nvtxs;i++) {
764:     PetscSortRemoveDupsInt(&graph->count[i],graph->neighbours_set[i]);
765:   }
766:   /* Get info for dofs splitting */
767:   for (i=0;i<n_ISForDofs;i++) {
768:     ISGetSize(ISForDofs[i],&is_size);
769:     ISGetIndices(ISForDofs[i],(const PetscInt**)&is_indices);
770:     for (j=0;j<is_size;j++) {
771:       graph->which_dof[is_indices[j]]=i;
772:     }
773:     ISRestoreIndices(ISForDofs[i],(const PetscInt**)&is_indices);
774:   }
775:   /* Take into account Neumann nodes */
776:   VecSet(local_vec,0.0);
777:   VecSet(local_vec2,0.0);
778:   if (neumann_is) {
779:     VecGetArray(local_vec,&array);
780:     ISGetSize(neumann_is,&is_size);
781:     ISGetIndices(neumann_is,(const PetscInt**)&is_indices);
782:     for (i=0;i<is_size;i++) {
783:       array[is_indices[i]] = 1.0;
784:     }
785:     ISRestoreIndices(neumann_is,(const PetscInt**)&is_indices);
786:     VecRestoreArray(local_vec,&array);
787:   }
788:   /* Neumann nodes: impose consistency among neighbours */
789:   VecSet(global_vec,0.0);
790:   VecScatterBegin(scatter_ctx,local_vec,global_vec,ADD_VALUES,SCATTER_REVERSE);
791:   VecScatterEnd(scatter_ctx,local_vec,global_vec,ADD_VALUES,SCATTER_REVERSE);
792:   VecScatterBegin(scatter_ctx,global_vec,local_vec,INSERT_VALUES,SCATTER_FORWARD);
793:   VecScatterEnd(scatter_ctx,global_vec,local_vec,INSERT_VALUES,SCATTER_FORWARD);
794:   VecGetArray(local_vec,&array);
795:   for (i=0;i<graph->nvtxs;i++) {
796:     if (PetscRealPart(array[i]) > 0.0) {
797:       graph->special_dof[i] = NEUMANN_MARK;
798:     }
799:   }
800:   VecRestoreArray(local_vec,&array);
801:   /* Take into account Dirichlet nodes */
802:   VecSet(local_vec2,0.0);
803:   if (dirichlet_is) {
804:     VecGetArray(local_vec,&array);
805:     VecGetArray(local_vec2,&array2);
806:     ISGetSize(dirichlet_is,&is_size);
807:     ISGetIndices(dirichlet_is,(const PetscInt**)&is_indices);
808:     for (i=0;i<is_size;i++){
809:       k = is_indices[i];
810:       if (graph->count[k] && !PetscBTLookup(graph->touched,k)) {
811:         if (PetscRealPart(array[k]) > 0.0) {
812:           SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"BDDC cannot have boundary nodes which are marked Neumann and Dirichlet at the same time! Local node %d is wrong!\n",k);
813:         }
814:         array2[k] = 1.0;
815:       }
816:     }
817:     ISRestoreIndices(dirichlet_is,(const PetscInt**)&is_indices);
818:     VecRestoreArray(local_vec,&array);
819:     VecRestoreArray(local_vec2,&array2);
820:   }
821:   /* Dirichlet nodes: impose consistency among neighbours */
822:   VecSet(global_vec,0.0);
823:   VecScatterBegin(scatter_ctx,local_vec2,global_vec,ADD_VALUES,SCATTER_REVERSE);
824:   VecScatterEnd(scatter_ctx,local_vec2,global_vec,ADD_VALUES,SCATTER_REVERSE);
825:   VecScatterBegin(scatter_ctx,global_vec,local_vec,INSERT_VALUES,SCATTER_FORWARD);
826:   VecScatterEnd(scatter_ctx,global_vec,local_vec,INSERT_VALUES,SCATTER_FORWARD);
827:   VecGetArray(local_vec,&array);
828:   for (i=0;i<graph->nvtxs;i++) {
829:     if (PetscRealPart(array[i]) > 0.0) {
830:       PetscBTSet(graph->touched,i);
831:       graph->subset[i] = 0; /* dirichlet nodes treated as internal -> is it ok? */
832:       graph->special_dof[i] = DIRICHLET_MARK;
833:     }
834:   }
835:   VecRestoreArray(local_vec,&array);

837:   /* mark local periodic nodes (if any) and adapt CSR graph */
838:   if (graph->mirrors) {
839:     PetscInt *new_xadj,*new_adjncy;

841:     for (i=0;i<graph->nvtxs;i++)
842:       if (graph->mirrors[i])
843:         graph->special_dof[i] = LOCAL_PERIODIC_MARK;

845:     /* sort CSR graph */
846:     for (i=0;i<graph->nvtxs;i++)
847:       PetscSortInt(graph->xadj[i+1]-graph->xadj[i],&graph->adjncy[graph->xadj[i]]);

849:     /* adapt local CSR graph in case of local periodicity */
850:     k=0;
851:     for (i=0;i<graph->nvtxs;i++)
852:       for (j=graph->xadj[i];j<graph->xadj[i+1];j++)
853:         k += graph->mirrors[graph->adjncy[j]];

855:     PetscMalloc1((graph->nvtxs+1),&new_xadj);
856:     PetscMalloc1((k+graph->xadj[graph->nvtxs]),&new_adjncy);
857:     new_xadj[0]=0;
858:     for (i=0;i<graph->nvtxs;i++) {
859:       k = graph->xadj[i+1]-graph->xadj[i];
860:       PetscMemcpy(&new_adjncy[new_xadj[i]],&graph->adjncy[graph->xadj[i]],k*sizeof(PetscInt));
861:       new_xadj[i+1]=new_xadj[i]+k;
862:       for (j=graph->xadj[i];j<graph->xadj[i+1];j++) {
863:         k = graph->mirrors[graph->adjncy[j]];
864:         PetscMemcpy(&new_adjncy[new_xadj[i+1]],graph->mirrors_set[graph->adjncy[j]],k*sizeof(PetscInt));
865:         new_xadj[i+1]+=k;
866:       }
867:       k = new_xadj[i+1]-new_xadj[i];
868:       PetscSortRemoveDupsInt(&k,&new_adjncy[new_xadj[i]]);
869:       new_xadj[i+1]=new_xadj[i]+k;
870:     }
871:     /* set new CSR into graph */
872:     PetscFree(graph->xadj);
873:     PetscFree(graph->adjncy);
874:     graph->xadj = new_xadj;
875:     graph->adjncy = new_adjncy;
876:   }

878:   /* mark special nodes -> each will become a single node equivalence class */
879:   if (custom_primal_vertices) {
880:     ISGetSize(custom_primal_vertices,&is_size);
881:     ISGetIndices(custom_primal_vertices,(const PetscInt**)&is_indices);
882:     for (i=0;i<is_size;i++) {
883:       graph->special_dof[is_indices[i]] = SPECIAL_MARK-i;
884:     }
885:     ISRestoreIndices(custom_primal_vertices,(const PetscInt**)&is_indices);
886:   }
887:   /* mark interior nodes as touched and belonging to partition number 0 */
888:   for (i=0;i<graph->nvtxs;i++) {
889:     if (!graph->count[i]) {
890:       PetscBTSet(graph->touched,i);
891:       graph->subset[i] = 0;
892:     }
893:   }
894:   /* init graph structure and compute default subsets */
895:   nodes_touched=0;
896:   for (i=0;i<graph->nvtxs;i++) {
897:     if (PetscBTLookup(graph->touched,i)) {
898:       nodes_touched++;
899:     }
900:   }
901:   i = 0;
902:   graph->ncc = 0;
903:   total_counts = 0;
904:   while (nodes_touched<graph->nvtxs) {
905:     /*  find first untouched node in local ordering */
906:     while (PetscBTLookup(graph->touched,i)) {
907:       i++;
908:     }
909:     PetscBTSet(graph->touched,i);
910:     graph->subset[i] = graph->ncc+1;
911:     graph->cptr[graph->ncc] = total_counts;
912:     graph->queue[total_counts] = i;
913:     total_counts++;
914:     nodes_touched++;
915:     /* now find all other nodes having the same set of sharing subdomains */
916:     for (j=i+1;j<graph->nvtxs;j++) {
917:       /* check for same number of sharing subdomains, dof number and same special mark */
918:       if (!PetscBTLookup(graph->touched,j) && graph->count[i] == graph->count[j] && graph->which_dof[i] == graph->which_dof[j] && graph->special_dof[i] == graph->special_dof[j]) {
919:         /* check for same set of sharing subdomains */
920:         same_set=PETSC_TRUE;
921:         for (k=0;k<graph->count[j];k++){
922:           if (graph->neighbours_set[i][k]!=graph->neighbours_set[j][k]) {
923:             same_set=PETSC_FALSE;
924:           }
925:         }
926:         /* I found a friend of mine */
927:         if (same_set) {
928:           graph->subset[j]=graph->ncc+1;
929:           PetscBTSet(graph->touched,j);
930:           nodes_touched++;
931:           graph->queue[total_counts] = j;
932:           total_counts++;
933:         }
934:       }
935:     }
936:     graph->ncc++;
937:   }
938:   /* set default number of subsets (at this point no info on csr graph has been taken into account, so n_subsets = ncc */
939:   graph->n_subsets = graph->ncc;
940:   PetscMalloc1(graph->n_subsets,&graph->subset_ncc);
941:   for (i=0;i<graph->n_subsets;i++) {
942:     graph->subset_ncc[i] = 1;
943:   }
944:   /* final pointer */
945:   graph->cptr[graph->ncc] = total_counts;
946:   /* free memory allocated by ISLocalToGlobalMappingGetInfo */
947:   ISLocalToGlobalMappingRestoreInfo(graph->l2gmap,&n_neigh,&neigh,&n_shared,&shared);
948:   /* free objects */
949:   VecDestroy(&local_vec);
950:   VecDestroy(&local_vec2);
951:   VecDestroy(&global_vec);
952:   VecScatterDestroy(&scatter_ctx);
953:   return(0);
954: }

958: PetscErrorCode PCBDDCGraphResetCSR(PCBDDCGraph graph)
959: {

963:   PetscFree(graph->xadj);
964:   PetscFree(graph->adjncy);
965:   graph->nvtxs_csr = 0;
966:   return(0);
967: }

971: PetscErrorCode PCBDDCGraphReset(PCBDDCGraph graph)
972: {

976:   ISLocalToGlobalMappingDestroy(&graph->l2gmap);
977:   PetscFree(graph->subset_ncc);
978:   if (graph->nvtxs) {
979:     PetscFree(graph->neighbours_set[0]);
980:   }
981:   PetscBTDestroy(&graph->touched);
982:   PetscFree7(graph->count,
983:                     graph->neighbours_set,
984:                     graph->subset,
985:                     graph->which_dof,
986:                     graph->cptr,
987:                     graph->queue,
988:                     graph->special_dof);
989:   if (graph->mirrors) {
990:     PetscFree(graph->mirrors_set[0]);
991:   }
992:   PetscFree2(graph->mirrors,graph->mirrors_set);
993:   graph->nvtxs = 0;
994:   graph->n_subsets = 0;
995:   graph->custom_minimal_size = 1;
996:   return(0);
997: }

1001: PetscErrorCode PCBDDCGraphInit(PCBDDCGraph graph, ISLocalToGlobalMapping l2gmap)
1002: {
1003:   PetscInt       n;

1009:   /* raise an error if already allocated */
1010:   if (graph->nvtxs) {
1011:     SETERRQ(PetscObjectComm((PetscObject)l2gmap),PETSC_ERR_PLIB,"BDDCGraph already initialized");
1012:   }
1013:   /* set number of vertices */
1014:   PetscObjectReference((PetscObject)l2gmap);
1015:   graph->l2gmap = l2gmap;
1016:   ISLocalToGlobalMappingGetSize(l2gmap,&n);
1017:   graph->nvtxs = n;
1018:   /* allocate used space */
1019:   PetscBTCreate(graph->nvtxs,&graph->touched);
1020:   PetscMalloc7(graph->nvtxs,&graph->count,
1021:                       graph->nvtxs,&graph->neighbours_set,
1022:                       graph->nvtxs,&graph->subset,
1023:                       graph->nvtxs,&graph->which_dof,
1024:                       graph->nvtxs+1,&graph->cptr,
1025:                       graph->nvtxs,&graph->queue,
1026:                       graph->nvtxs,&graph->special_dof);
1027:   /* zeroes memory */
1028:   PetscMemzero(graph->count,graph->nvtxs*sizeof(PetscInt));
1029:   PetscMemzero(graph->subset,graph->nvtxs*sizeof(PetscInt));
1030:   PetscMemzero(graph->which_dof,graph->nvtxs*sizeof(PetscInt));
1031:   PetscMemzero(graph->cptr,(graph->nvtxs+1)*sizeof(PetscInt));
1032:   PetscMemzero(graph->queue,graph->nvtxs*sizeof(PetscInt));
1033:   PetscMemzero(graph->special_dof,graph->nvtxs*sizeof(PetscInt));
1034:   /* zeroes first pointer to neighbour set */
1035:   if (graph->nvtxs) {
1036:     graph->neighbours_set[0] = 0;
1037:   }
1038:   /* zeroes workspace for values of ncc */
1039:   graph->subset_ncc = 0;
1040:   return(0);
1041: }

1045: PetscErrorCode PCBDDCGraphDestroy(PCBDDCGraph* graph)
1046: {

1050:   PCBDDCGraphReset(*graph);
1051:   PetscFree(*graph);
1052:   return(0);
1053: }

1057: PetscErrorCode PCBDDCGraphCreate(PCBDDCGraph *graph)
1058: {
1059:   PCBDDCGraph    new_graph;

1063:   PetscMalloc(sizeof(*new_graph),&new_graph);
1064:   /* local to global mapping of dofs */
1065:   new_graph->l2gmap = 0;
1066:   /* vertex size */
1067:   new_graph->nvtxs = 0;
1068:   new_graph->n_subsets = 0;
1069:   new_graph->custom_minimal_size = 1;
1070:   /* zeroes ponters */
1071:   new_graph->mirrors = 0;
1072:   new_graph->mirrors_set = 0;
1073:   new_graph->neighbours_set = 0;
1074:   new_graph->subset = 0;
1075:   new_graph->which_dof = 0;
1076:   new_graph->special_dof = 0;
1077:   new_graph->cptr = 0;
1078:   new_graph->queue = 0;
1079:   new_graph->count = 0;
1080:   new_graph->subset_ncc = 0;
1081:   new_graph->touched = 0;
1082:   /* zeroes pointers to csr graph of local nodes connectivity (optional data) */
1083:   new_graph->nvtxs_csr = 0;
1084:   new_graph->xadj = 0;
1085:   new_graph->adjncy = 0;
1086:   *graph = new_graph;
1087:   return(0);
1088: }