Actual source code: Distribution.hh

  1: #ifndef included_ALE_Distribution_hh
  2: #define included_ALE_Distribution_hh

  4: #ifndef  included_ALE_Mesh_hh
  5: #include <Mesh.hh>
  6: #endif

  8: #ifndef  included_ALE_Partitioner_hh
  9: #include <Partitioner.hh>
 10: #endif

 12: #ifndef  included_ALE_Completion_hh
 13: #include <Completion.hh>
 14: #endif

 16: // Attempt to unify all of the distribution mechanisms:
 17: //   one to many  (distributeMesh)
 18: //   many to one  (unifyMesh)
 19: //   many to many (Numbering)
 20: // as well as things being distributed
 21: //   Section
 22: //   Sieve        (This sends two sections, the points and cones)
 23: //   Numbering    (Should be an integer section)
 24: //   Global Order (should be an integer section with extra methods)
 25: //
 26: // 0) Create the new object to hold the communicated data
 27: //
 28: // 1) Create Overlap
 29: //    There may be special ways to do this based upon what we know at the time
 30: //
 31: // 2) Create send and receive sections over the interface
 32: //    These have a flat topology now, consisting only of the overlap nodes
 33: //    We could make a full topology on the overlap (maybe it is necessary for higher order)
 34: //
 35: // 3) Communication section
 36: //    Create sizer sections on interface (uses constant sizer)
 37: //    Communicate sizes on interface (uses custom filler)
 38: //      Fill send section
 39: //      sendSection->startCommunication();
 40: //      recvSection->startCommunication();
 41: //      sendSection->endCommunication();
 42: //      recvSection->endCommunication();
 43: //
 44: //    Create section on interface (uses previous sizer)
 45: //    Communicate values on interface (uses custom filler)
 46: //      Same stuff as above
 47: //
 48: // 4) Update new section with old local values (can be done in between the communication?)
 49: //    Loop over patches in new topology
 50: //      Loop over chart from patch in old atlas
 51: //        If this point is in the new sieve from patch
 52: //          Set to old fiber dimension
 53: //    Order and allocate new section
 54: //    Repeat loop, but update values
 55: //
 56: // 5) Update new section with old received values
 57: //    Loop over patches in discrete topology of receive section (these are ranks)
 58: //      Loop over base of discrete sieve (we should transform this to a chart to match above)
 59: //        Get new patch from overlap, or should the receive patches be <rank, patch>?
 60: //        Guaranteed to be in the new sieve from patch (but we could check anyway)
 61: //          Set to recevied fiber dimension
 62: //    Order and allocate new section
 63: //    Repeat loop, but update values
 64: //
 65: // 6) Synchronize PETSc tags (can I get around this?)
 66: namespace ALE {
 67:   template<typename Bundle_>
 68:   class Distribution {
 69:   public:
 70:     typedef Bundle_                                                                     bundle_type;
 71:     typedef typename bundle_type::sieve_type                                            sieve_type;
 72:     typedef typename bundle_type::point_type                                            point_type;
 73:     typedef typename bundle_type::alloc_type                                            alloc_type;
 74:     typedef typename ALE::New::Completion<bundle_type, typename sieve_type::point_type>                            sieveCompletion;
 75:     typedef typename ALE::New::SectionCompletion<bundle_type, typename bundle_type::real_section_type::value_type> sectionCompletion;
 76:     typedef typename sectionCompletion::send_overlap_type                               send_overlap_type;
 77:     typedef typename sectionCompletion::recv_overlap_type                               recv_overlap_type;
 78:   public:
 81:     static void createPartitionOverlap(const Obj<bundle_type>& bundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
 82:       const Obj<send_overlap_type>& topSendOverlap = bundle->getSendOverlap();
 83:       const Obj<recv_overlap_type>& topRecvOverlap = bundle->getRecvOverlap();
 84:       const Obj<typename send_overlap_type::traits::baseSequence> base = topSendOverlap->base();
 85:       const Obj<typename recv_overlap_type::traits::capSequence>  cap  = topRecvOverlap->cap();
 86:       const int rank = bundle->commRank();

 88:       if (base->empty()) {
 89:         if (rank == 0) {
 90:           for(int p = 1; p < bundle->commSize(); p++) {
 91:             // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
 92:             sendOverlap->addCone(p, p, p);
 93:           }
 94:         }
 95:       } else {
 96:         for(typename send_overlap_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
 97:           const int& p = *b_iter;
 98:           // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
 99:           sendOverlap->addCone(p, p, p);
100:         }
101:       }
102:       if (cap->empty()) {
103:         if (rank != 0) {
104:           // The arrow is from local partition point rank (color) on rank 0 (source) to remote partition point rank (target)
105:           recvOverlap->addCone(0, rank, rank);
106:         }
107:       } else {
108:         for(typename recv_overlap_type::traits::capSequence::iterator c_iter = cap->begin(); c_iter != cap->end(); ++c_iter) {
109:           const int& p = *c_iter;
110:           // The arrow is from local partition point rank (color) on rank p (source) to remote partition point rank (target)
111:           recvOverlap->addCone(p, rank, rank);
112:         }
113:       }
114:     };
117:     template<typename Partitioner>
118:     static typename Partitioner::part_type *createAssignment(const Obj<bundle_type>& bundle, const int dim, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const int height = 0) {
119:       // 1) Form partition point overlap a priori
120:       createPartitionOverlap(bundle, sendOverlap, recvOverlap);
121:       if (bundle->debug()) {
122:         sendOverlap->view("Send overlap for partition");
123:         recvOverlap->view("Receive overlap for partition");
124:       }
125:       // 2) Partition the mesh
126:       if (height == 0) {
127:         return Partitioner::partitionSieve(bundle, dim);
128:       } else if (height == 1) {
129:         return Partitioner::partitionSieveByFace(bundle, dim);
130:       }
131:       throw ALE::Exception("Invalid partition height");
132:     };
135:     // Partition a bundle on process 0 and scatter to all processes
136:     static void scatterBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const std::string& partitioner, const int height = 0, const Obj<bundle_type>& subBundle = NULL, const Obj<bundle_type>& subBundleNew = NULL) {
137:       if (height == 0) {
138:         if (partitioner == "chaco") {
139: #ifdef PETSC_HAVE_CHACO
140:           typedef typename ALE::New::Chaco::Partitioner<bundle_type> Partitioner;
141:           typedef typename ALE::New::Partitioner<bundle_type>        GenPartitioner;
142:           typedef typename Partitioner::part_type                    part_type;

144:           part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
145:           if (!subBundle.isNull() && !subBundleNew.isNull()) {
146:             part_type *subAssignment = GenPartitioner::subordinatePartition(bundle, 1, subBundle, assignment);
147:             const Obj<sieve_type>& sieve      = subBundle->getSieve();
148:             const Obj<sieve_type>& sieveNew   = new Mesh::sieve_type(subBundle->comm(), subBundle->debug());
149:             const int              numCells   = subBundle->heightStratum(height)->size();

151:             subBundleNew->setSieve(sieveNew);
152:             sieveCompletion::scatterSieve(subBundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numCells, subAssignment);
153:             subBundleNew->stratify();
154:             if (subAssignment != NULL) delete [] subAssignment;
155:           }
156:           if (assignment != NULL) delete [] assignment;
157: #else
158:           throw ALE::Exception("Chaco is not installed. Reconfigure with the flag --download-chaco");
159: #endif
160:         } else if (partitioner == "parmetis") {
161: #ifdef PETSC_HAVE_PARMETIS
162:           typedef typename ALE::New::ParMetis::Partitioner<bundle_type> Partitioner;
163:           typedef typename ALE::New::Partitioner<bundle_type>           GenPartitioner;
164:           typedef typename Partitioner::part_type                       part_type;

166:           part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
167:           if (!subBundle.isNull() && !subBundleNew.isNull()) {
168:             part_type *subAssignment = GenPartitioner::subordinatePartition(bundle, 1, subBundle, assignment);
169:             const Obj<sieve_type>& sieve      = subBundle->getSieve();
170:             const Obj<sieve_type>& sieveNew   = new Mesh::sieve_type(subBundle->comm(), subBundle->debug());
171:             const int              numCells   = subBundle->heightStratum(height)->size();

173:             subBundleNew->setSieve(sieveNew);
174:             sieveCompletion::scatterSieve(subBundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numCells, subAssignment);
175:             subBundleNew->stratify();
176:             if (subAssignment != NULL) delete [] subAssignment;
177:           }
178:           if (assignment != NULL) delete [] assignment;
179: #else
180:           throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
181: #endif
182:         } else {
183:           throw ALE::Exception("Unknown partitioner");
184:         }
185:       } else if (height == 1) {
186:         if (partitioner == "zoltan") {
187: #ifdef PETSC_HAVE_ZOLTAN
188:           typedef typename ALE::New::Zoltan::Partitioner<bundle_type> Partitioner;
189:           typedef typename Partitioner::part_type                     part_type;

191:           part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
192:           if (assignment != NULL) delete [] assignment;
193: #else
194:           throw ALE::Exception("Zoltan is not installed. Reconfigure with the flag --download-zoltan");
195: #endif
196:         } else if (partitioner == "parmetis") {
197: #ifdef PETSC_HAVE_PARMETIS
198:           typedef typename ALE::New::ParMetis::Partitioner<bundle_type> Partitioner;
199:           typedef typename Partitioner::part_type                       part_type;

201:           part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
202:           if (assignment != NULL) delete [] assignment;
203: #else
204:           throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
205: #endif
206:         } else {
207:           throw ALE::Exception("Unknown partitioner");
208:         }
209:       } else {
210:         throw ALE::Exception("Invalid partition height");
211:       }
212:     };
213:     template<typename Partitioner>
214:     static typename Partitioner::part_type *scatterBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const int height = 0) {
215:       typename Partitioner::part_type *assignment = createAssignment<Partitioner>(bundle, dim, sendOverlap, recvOverlap, height);
216:       const Obj<sieve_type>&           sieve      = bundle->getSieve();
217:       const Obj<sieve_type>&           sieveNew   = bundleNew->getSieve();
218:       const int                        numPoints  = bundle->heightStratum(height)->size();

220:       sieveCompletion::scatterSieve(bundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numPoints, assignment);
221:       bundleNew->stratify();
222:       return assignment;
223:     };
226:     static Obj<ALE::Mesh> distributeMesh(const Obj<ALE::Mesh>& serialMesh, const int height = 0, const std::string& partitioner = "chaco") {
227:       MPI_Comm                          comm          = serialMesh->comm();
228:       const int                         dim           = serialMesh->getDimension();
229:       Obj<ALE::Mesh>                    parallelMesh  = new ALE::Mesh(comm, dim, serialMesh->debug());
230:       const Obj<ALE::Mesh::sieve_type>& parallelSieve = new ALE::Mesh::sieve_type(comm, serialMesh->debug());

232:       ALE_LOG_EVENT_BEGIN;
233:       parallelMesh->setSieve(parallelSieve);
234:       if (serialMesh->debug()) {serialMesh->view("Serial mesh");}

236:       // Distribute cones
237:       Obj<send_overlap_type> sendOverlap = new send_overlap_type(comm, serialMesh->debug());
238:       Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(comm, serialMesh->debug());
239:       scatterBundle(serialMesh, dim, parallelMesh, sendOverlap, recvOverlap, partitioner, height);
240:       parallelMesh->setDistSendOverlap(sendOverlap);
241:       parallelMesh->setDistRecvOverlap(recvOverlap);

243:       // Distribute labels
244:       const typename bundle_type::labels_type& labels = serialMesh->getLabels();

246:       for(typename bundle_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
247:         if (parallelMesh->hasLabel(l_iter->first)) continue;
248:         const Obj<typename bundle_type::label_type>& serialLabel   = l_iter->second;
249:         const Obj<typename bundle_type::label_type>& parallelLabel = parallelMesh->createLabel(l_iter->first);
250:         // Create local label
251: #define NEW_LABEL
252: #ifdef NEW_LABEL
253:         parallelLabel->add(serialLabel, parallelSieve);
254: #else
255:         const Obj<typename bundle_type::label_type::traits::baseSequence>& base = serialLabel->base();

257:         for(typename bundle_type::label_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
258:           if (parallelSieve->capContains(*b_iter) || parallelSieve->baseContains(*b_iter)) {
259:             parallelLabel->addArrow(*serialLabel->cone(*b_iter)->begin(), *b_iter);
260:           }
261:         }
262: #endif
263:         // Get remote labels
264:         sieveCompletion::scatterCones(serialLabel, parallelLabel, sendOverlap, recvOverlap);
265:       }

267:       // Distribute sections
268:       Obj<std::set<std::string> > sections = serialMesh->getRealSections();

270:       for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
271:         parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), parallelMesh, sendOverlap, recvOverlap));
272:       }
273:       sections = serialMesh->getIntSections();
274:       for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
275:         parallelMesh->setIntSection(*name, distributeSection(serialMesh->getIntSection(*name), parallelMesh, sendOverlap, recvOverlap));
276:       }
277:       sections = serialMesh->getArrowSections();

279:       for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
280:         parallelMesh->setArrowSection(*name, distributeArrowSection(serialMesh->getArrowSection(*name), serialMesh, parallelMesh, sendOverlap, recvOverlap));
281:       }
282:       if (parallelMesh->debug()) {parallelMesh->view("Parallel Mesh");}
283:       ALE_LOG_EVENT_END;
284:       return parallelMesh;
285:     };
288:     template<typename Section>
289:     static void updateSectionLocal(const Obj<Section>& oldSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
290:       const Obj<typename bundle_type::sieve_type>&    newSieve = newBundle->getSieve();
291:       const typename Section::atlas_type::chart_type& oldChart = oldSection->getChart();

293:       for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
294:         if (newSieve->capContains(*c_iter) || newSieve->baseContains(*c_iter)) {
295:           newSection->setFiberDimension(*c_iter, oldSection->getFiberDimension(*c_iter));
296:         }
297:       }
298:       newBundle->allocate(newSection);
299:       const typename Section::atlas_type::chart_type& newChart = newSection->getChart();

301:       for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
302:         newSection->updatePointAll(*c_iter, oldSection->restrictPoint(*c_iter));
303:       }
304:     };
307:     template<typename RecvSection, typename Section>
308:     static void updateSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
309:       Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();

311:       for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
312:         const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
313:         const typename recv_overlap_type::traits::coneSequence::iterator end         = recvPatches->end();

315:         for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
316:           newSection->addPoint(*r_iter, recvSection->getSection(*p_iter)->getFiberDimension(*r_iter));
317:         }
318:       }
319:       newBundle->reallocate(newSection);
320:       for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
321:         const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
322:         const typename recv_overlap_type::traits::coneSequence::iterator end         = recvPatches->end();

324:         for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
325:           if (recvSection->getSection(*p_iter)->getFiberDimension(*r_iter)) {
326:             newSection->updatePointAll(*r_iter, recvSection->getSection(*p_iter)->restrictPoint(*r_iter));
327:           }
328:         }
329:       }
330:     };
333:     template<typename Section>
334:     static Obj<Section> distributeSection(const Obj<Section>& serialSection, const Obj<bundle_type>& parallelBundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
335:       if (serialSection->debug()) {
336:         serialSection->view("Serial Section");
337:       }
338:       typedef typename alloc_type::template rebind<typename Section::value_type>::other value_alloc_type;
339:       typedef ALE::Field<send_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > send_section_type;
340:       typedef ALE::Field<recv_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > recv_section_type;
341:       typedef ALE::New::SizeSection<Section> SectionSizer;
342:       Obj<Section>                 parallelSection = new Section(serialSection->comm(), serialSection->debug());
343:       const Obj<send_section_type> sendSection     = new send_section_type(serialSection->comm(), serialSection->debug());
344:       const Obj<recv_section_type> recvSection     = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
345:       const Obj<SectionSizer>      sizer           = new SectionSizer(serialSection);

347:       updateSectionLocal(serialSection, parallelBundle, parallelSection);
348:       sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, serialSection, sendSection, recvSection);
349:       updateSectionRemote(recvOverlap, recvSection, parallelBundle, parallelSection);
350:       if (parallelSection->debug()) {
351:         parallelSection->view("Parallel Section");
352:       }
353:       return parallelSection;
354:     };
357:     template<typename Section>
358:     static void updateArrowSectionLocal(const Obj<Section>& oldSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
359:       const Obj<typename bundle_type::sieve_type>&    newSieve = newBundle->getSieve();
360:       const typename Section::atlas_type::chart_type& oldChart = oldSection->getChart();

362:       for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
363:         // Dmitry should provide a Sieve::contains(MinimalArrow) method
364:         if (newSieve->capContains(c_iter->source) && newSieve->baseContains(c_iter->target)) {
365:           newSection->setFiberDimension(*c_iter, oldSection->getFiberDimension(*c_iter));
366:         }
367:       }
368:       //newBundle->allocate(newSection);
369:       const typename Section::atlas_type::chart_type& newChart = newSection->getChart();

371:       for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
372:         newSection->updatePointAll(*c_iter, oldSection->restrictPoint(*c_iter));
373:       }
374:     };
377:     template<typename RecvSection, typename Section>
378:     static void updateArrowSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
379:       Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();

381:       for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
382:         const Obj<typename bundle_type::sieve_type::traits::coneSequence>&     cone = newBundle->getSieve()->cone(*r_iter);
383:         const typename bundle_type::sieve_type::traits::coneSequence::iterator end  = cone->end();

385:         for(typename bundle_type::sieve_type::traits::coneSequence::iterator c_iter = cone->begin(); c_iter != end; ++c_iter) {
386:           newSection->setFiberDimension(typename Section::point_type(*c_iter, *r_iter), 1);
387:         }
388:       }
389:       //newBundle->reallocate(newSection);
390:       for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
391:         const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
392:         const typename recv_overlap_type::traits::coneSequence::iterator recvEnd     = recvPatches->end();

394:         for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != recvEnd; ++p_iter) {
395:           const Obj<typename RecvSection::section_type>& section = recvSection->getSection(*p_iter);

397:           if (section->getFiberDimension(*r_iter)) {
398:             const Obj<typename bundle_type::sieve_type::traits::coneSequence>&     cone    = newBundle->getSieve()->cone(*r_iter);
399:             const typename bundle_type::sieve_type::traits::coneSequence::iterator end     = cone->end();
400:             const typename RecvSection::value_type                                *values  = section->restrictPoint(*r_iter);
401:             int                                                                    c       = -1;

403:             for(typename bundle_type::sieve_type::traits::coneSequence::iterator c_iter = cone->begin(); c_iter != end; ++c_iter) {
404:               newSection->updatePoint(typename Section::point_type(*c_iter, *r_iter), &values[++c]);
405:             }
406:           }
407:         }
408:       }
409:     };
412:     template<typename Section>
413:     static Obj<Section> distributeArrowSection(const Obj<Section>& serialSection, const Obj<bundle_type>& serialBundle, const Obj<bundle_type>& parallelBundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
414:       if (serialSection->debug()) {
415:         serialSection->view("Serial ArrowSection");
416:       }
417:       typedef typename alloc_type::template rebind<typename Section::value_type>::other value_alloc_type;
418:       typedef ALE::Field<send_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > send_section_type;
419:       typedef ALE::Field<recv_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > recv_section_type;
420:       typedef ALE::New::ConeSizeSection<bundle_type, sieve_type> SectionSizer;
421:       typedef ALE::New::ArrowSection<sieve_type, Section>        ArrowFiller;
422:       Obj<Section>                 parallelSection = new Section(serialSection->comm(), serialSection->debug());
423:       const Obj<send_section_type> sendSection     = new send_section_type(serialSection->comm(), serialSection->debug());
424:       const Obj<recv_section_type> recvSection     = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
425:       const Obj<SectionSizer>      sizer           = new SectionSizer(serialBundle, serialBundle->getSieve());
426:       const Obj<ArrowFiller>       filler          = new ArrowFiller(serialBundle->getSieve(), serialSection);

428:       updateArrowSectionLocal(serialSection, parallelBundle, parallelSection);
429:       sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
430:       updateArrowSectionRemote(recvOverlap, recvSection, parallelBundle, parallelSection);
431:       if (parallelSection->debug()) {
432:         parallelSection->view("Parallel ArrowSection");
433:       }
434:       return parallelSection;
435:     };
436:     static void unifyBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
437:       typedef int part_type;
438:       const Obj<sieve_type>& sieve    = bundle->getSieve();
439:       const Obj<sieve_type>& sieveNew = bundleNew->getSieve();
440:       const int              rank     = bundle->commRank();
441:       const int              debug    = bundle->debug();

443:       // 1) Form partition point overlap a priori
444:       if (rank == 0) {
445:         for(int p = 1; p < sieve->commSize(); p++) {
446:           // The arrow is from remote partition point 0 on rank p to local partition point 0
447:           recvOverlap->addCone(p, 0, 0);
448:         }
449:       } else {
450:         // The arrow is from local partition point 0 to remote partition point 0 on rank 0
451:         sendOverlap->addCone(0, 0, 0);
452:       }
453:       if (debug) {
454:         sendOverlap->view("Send overlap for partition");
455:         recvOverlap->view("Receive overlap for partition");
456:       }
457:       // 2) Partition the mesh
458:       int        numCells   = bundle->heightStratum(0)->size();
459:       part_type *assignment = new part_type[numCells];

461:       for(int c = 0; c < numCells; ++c) {
462:         assignment[c] = 0;
463:       }
464:       // 3) Scatter the sieve
465:       sieveCompletion::scatterSieve(bundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, 0, numCells, assignment);
466:       bundleNew->stratify();
467:       // 4) Cleanup
468:       if (assignment != NULL) delete [] assignment;
469:     };
472:     static Obj<ALE::Mesh> unifyMesh(const Obj<ALE::Mesh>& parallelMesh) {
473:       const int                         dim         = parallelMesh->getDimension();
474:       Obj<ALE::Mesh>                    serialMesh  = new ALE::Mesh(parallelMesh->comm(), dim, parallelMesh->debug());
475:       const Obj<ALE::Mesh::sieve_type>& serialSieve = new ALE::Mesh::sieve_type(parallelMesh->comm(), parallelMesh->debug());

477:       ALE_LOG_EVENT_BEGIN;
478:       serialMesh->setSieve(serialSieve);
479:       if (parallelMesh->debug()) {
480:         parallelMesh->view("Parallel topology");
481:       }

483:       // Unify cones
484:       Obj<send_overlap_type> sendOverlap = new send_overlap_type(serialMesh->comm(), serialMesh->debug());
485:       Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(serialMesh->comm(), serialMesh->debug());
486:       unifyBundle(parallelMesh, dim, serialMesh, sendOverlap, recvOverlap);
487:       serialMesh->setDistSendOverlap(sendOverlap);
488:       serialMesh->setDistRecvOverlap(recvOverlap);

490:       // Unify labels
491:       const typename bundle_type::labels_type& labels = parallelMesh->getLabels();

493:       for(typename bundle_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
494:         if (serialMesh->hasLabel(l_iter->first)) continue;
495:         const Obj<typename bundle_type::label_type>& parallelLabel = l_iter->second;
496:         const Obj<typename bundle_type::label_type>& serialLabel   = serialMesh->createLabel(l_iter->first);

498:         sieveCompletion::scatterCones(parallelLabel, serialLabel, sendOverlap, recvOverlap);
499:       }

501:       // Unify coordinates
502:       Obj<std::set<std::string> > sections = parallelMesh->getRealSections();

504:       for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
505:         serialMesh->setRealSection(*name, distributeSection(parallelMesh->getRealSection(*name), serialMesh, sendOverlap, recvOverlap));
506:       }
507:       sections = parallelMesh->getIntSections();
508:       for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
509:         serialMesh->setIntSection(*name, distributeSection(parallelMesh->getIntSection(*name), serialMesh, sendOverlap, recvOverlap));
510:       }
511:       sections = parallelMesh->getArrowSections();
512:       for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
513:         serialMesh->setArrowSection(*name, distributeArrowSection(parallelMesh->getArrowSection(*name), parallelMesh, serialMesh, sendOverlap, recvOverlap));
514:       }
515:       if (serialMesh->debug()) {serialMesh->view("Serial Mesh");}
516:       ALE_LOG_EVENT_END;
517:       return serialMesh;
518:     };
519:   public: // Do not like these
522:     // This is just crappy. We could introduce another phase to find out exactly what
523:     //   indices people do not have in the global order after communication
524:     template<typename SendSection, typename RecvSection>
525:     static void updateOverlap(const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
526:       const typename SendSection::sheaf_type& sendRanks = sendSection->getPatches();
527:       const typename RecvSection::sheaf_type& recvRanks = recvSection->getPatches();

529:       for(typename SendSection::sheaf_type::const_iterator p_iter = sendRanks.begin(); p_iter != sendRanks.end(); ++p_iter) {
530:         const typename SendSection::patch_type&               rank    = p_iter->first;
531:         const Obj<typename SendSection::section_type>&        section = p_iter->second;
532:         const typename SendSection::section_type::chart_type& chart   = section->getChart();

534:         for(typename SendSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
535:           const typename SendSection::value_type *points = section->restrictPoint(*b_iter);
536:           const int                               size   = section->getFiberDimension(*b_iter);

538:           for(int p = 0; p < size; p++) {
539:             sendOverlap->addArrow(points[p], rank, points[p]);
540:           }
541:         }
542:       }
543:       for(typename RecvSection::sheaf_type::const_iterator p_iter = recvRanks.begin(); p_iter != recvRanks.end(); ++p_iter) {
544:         const typename RecvSection::patch_type&               rank    = p_iter->first;
545:         const Obj<typename RecvSection::section_type>&        section = p_iter->second;
546:         const typename RecvSection::section_type::chart_type& chart   = section->getChart();

548:         for(typename RecvSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
549:           const typename RecvSection::value_type *points = section->restrictPoint(*b_iter);
550:           const int                               size   = section->getFiberDimension(*b_iter);

552:           for(int p = 0; p < size; p++) {
553:             recvOverlap->addArrow(rank, points[p], points[p]);
554:           }
555:         }
556:       }
557:     };
560:     template<typename RecvSection>
561:     static void updateSieve(const Obj<RecvSection>& recvSection, const Obj<sieve_type>& sieve) {
562:       const typename RecvSection::sheaf_type& ranks = recvSection->getPatches();

564:       for(typename RecvSection::sheaf_type::const_iterator p_iter = ranks.begin(); p_iter != ranks.end(); ++p_iter) {
565:         const Obj<typename RecvSection::section_type>&        section = p_iter->second;
566:         const typename RecvSection::section_type::chart_type& chart   = section->getChart();

568:         for(typename RecvSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
569:           const typename RecvSection::value_type *points = section->restrictPoint(*b_iter);
570:           int                                     size   = section->getFiberDimension(*b_iter);
571:           int                                     c      = 0;

573:           for(int p = 0; p < size; p++) {
574:             //sieve->addArrow(points[p], *b_iter, c++);
575:             sieve->addArrow(points[p], *b_iter, c);
576:           }
577:         }
578:       }
579:     };
582:     template<typename SendSection, typename RecvSection>
583:     static void coneCompletion(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const Obj<bundle_type>& bundle, const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection) {
584:       if (sendOverlap->commSize() == 1) return;
585:       // Distribute cones
586:       const Obj<sieve_type>&                                 sieve           = bundle->getSieve();
587:       const Obj<typename sieveCompletion::topology_type>     secTopology     = sieveCompletion::completion::createSendTopology(sendOverlap);
588:       const Obj<typename sieveCompletion::cone_size_section> coneSizeSection = new typename sieveCompletion::cone_size_section(bundle, sieve);
589:       const Obj<typename sieveCompletion::cone_section>      coneSection     = new typename sieveCompletion::cone_section(sieve);
590:       sieveCompletion::completion::completeSection(sendOverlap, recvOverlap, coneSizeSection, coneSection, sendSection, recvSection);
591:       // Update cones
592:       updateSieve(recvSection, sieve);
593:     };
596:     template<typename Section>
597:     static void completeSection(const Obj<bundle_type>& bundle, const Obj<Section>& section) {
598:       typedef typename Distribution<bundle_type>::sieveCompletion sieveCompletion;
599:       typedef typename bundle_type::send_overlap_type             send_overlap_type;
600:       typedef typename bundle_type::recv_overlap_type             recv_overlap_type;
601:       typedef typename Section::value_type                        value_type;
602:       typedef typename alloc_type::template rebind<typename Section::value_type>::other value_alloc_type;
603:       typedef typename ALE::Field<send_overlap_type, int, ALE::Section<point_type, value_type, value_alloc_type> > send_section_type;
604:       typedef typename ALE::Field<recv_overlap_type, int, ALE::Section<point_type, value_type, value_alloc_type> > recv_section_type;
605:       typedef ALE::New::SizeSection<Section>                                SectionSizer;
606:       const int debug = section->debug();

608:       bundle->constructOverlap();
609:       const Obj<send_overlap_type> sendOverlap = bundle->getSendOverlap();
610:       const Obj<recv_overlap_type> recvOverlap = bundle->getRecvOverlap();
611:       const Obj<send_section_type> sendSection = new send_section_type(section->comm(), section->debug());
612:       const Obj<recv_section_type> recvSection = new recv_section_type(section->comm(), sendSection->getTag(), section->debug());
613:       const Obj<SectionSizer>      sizer       = new SectionSizer(section);

615:       sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, section, sendSection, recvSection);
616:       // Update section with remote data
617:       const Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = bundle->getRecvOverlap()->base();

619:       for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
620:         const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
621:         const typename recv_overlap_type::traits::coneSequence::iterator end         = recvPatches->end();

623:         for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
624:           if (recvSection->getSection(*p_iter)->getFiberDimension(p_iter.color())) {
625:             if (debug) {std::cout << "["<<section->commRank()<<"]Completed point " << *r_iter << std::endl;}
626:             section->updateAddPoint(*r_iter, recvSection->getSection(*p_iter)->restrictPoint(p_iter.color()));
627:           }
628:         }
629:       }
630:     };
631:   };
632: }

634: namespace ALECompat {
635:   namespace New {
636:     template<typename Topology_>
637:     class Distribution {
638:     public:
639:       typedef Topology_                                                                   topology_type;
640:       typedef typename topology_type::sieve_type                                          sieve_type;
641:       typedef ALECompat::New::Completion<Topology_, Mesh::sieve_type::point_type>               sieveCompletion;
642:       typedef ALECompat::New::SectionCompletion<Topology_, Mesh::real_section_type::value_type> sectionCompletion;
643:       typedef typename sectionCompletion::send_overlap_type                               send_overlap_type;
644:       typedef typename sectionCompletion::recv_overlap_type                               recv_overlap_type;
645:     public:
648:       // This is just crappy. WE could introduce another phase to find out exactly what
649:       //   indices people do not have in the global order after communication
650:       template<typename SendSection, typename RecvSection>
651:       static void updateOverlap(const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
652:         const typename SendSection::topology_type::sheaf_type& sendRanks = sendSection->getTopology()->getPatches();
653:         const typename RecvSection::topology_type::sheaf_type& recvRanks = recvSection->getTopology()->getPatches();

655:         for(typename SendSection::topology_type::sheaf_type::const_iterator p_iter = sendRanks.begin(); p_iter != sendRanks.end(); ++p_iter) {
656:           int                                                                       rank = p_iter->first;
657:           const Obj<typename SendSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();

659:           for(typename SendSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
660:             const typename SendSection::value_type *points = sendSection->restrict(rank, *b_iter);
661:             int size = sendSection->getFiberDimension(rank, *b_iter);

663:             for(int p = 0; p < size; p++) {
664:               sendOverlap->addArrow(points[p], rank, points[p]);
665:             }
666:           }
667:         }
668:         for(typename RecvSection::topology_type::sheaf_type::const_iterator p_iter = recvRanks.begin(); p_iter != recvRanks.end(); ++p_iter) {
669:           int                                                                       rank = p_iter->first;
670:           const Obj<typename RecvSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();

672:           for(typename RecvSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
673:             const typename RecvSection::value_type *points = recvSection->restrict(rank, *b_iter);
674:             int size = recvSection->getFiberDimension(rank, *b_iter);

676:             for(int p = 0; p < size; p++) {
677:               recvOverlap->addArrow(rank, points[p], points[p]);
678:             }
679:           }
680:         }
681:       };
682:       static void createLabelOverlap(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
683:       };
686:       template<typename Section>
687:       static void updateSectionLocal(const Obj<Section>& oldSection, const Obj<Section>& newSection)
688:       {
689:         const typename Section::topology_type::sheaf_type& patches = newSection->getTopology()->getPatches();

691:         for(typename Section::topology_type::sheaf_type::const_iterator p_iter = patches.begin(); p_iter != patches.end(); ++p_iter) {
692:           const typename Section::patch_type&                     patch    = p_iter->first;
693:           const Obj<typename Section::topology_type::sieve_type>& newSieve = p_iter->second;
694:           if (!oldSection->hasPatch(patch)) continue;
695:           const typename Section::atlas_type::chart_type&         oldChart = oldSection->getPatch(patch);

697:           for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
698:             if (newSieve->hasPoint(*c_iter)) {
699:               newSection->setFiberDimension(patch, *c_iter, oldSection->getFiberDimension(patch, *c_iter));
700:             }
701:           }
702:         }
703:         newSection->allocate();
704:         for(typename Section::topology_type::sheaf_type::const_iterator p_iter = patches.begin(); p_iter != patches.end(); ++p_iter) {
705:           const typename Section::patch_type&             patch    = p_iter->first;
706:           if (!oldSection->hasPatch(patch)) continue;
707:           const typename Section::atlas_type::chart_type& newChart = newSection->getPatch(patch);

709:           for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
710:             newSection->updatePoint(patch, *c_iter, oldSection->restrictPoint(patch, *c_iter));
711:           }
712:         }
713:       };
716:       template<typename RecvSection, typename Section>
717:       static void updateSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<Section>& newSection) {
718:         const Mesh::real_section_type::patch_type                  patch      = 0;
719:         Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();

721:         for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
722:           const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
723:           const typename recv_overlap_type::traits::coneSequence::iterator end         = recvPatches->end();

725:           for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
726:             newSection->addPoint(patch, *r_iter, recvSection->getFiberDimension(*p_iter, *r_iter));
727:           }
728:         }
729:         newSection->reallocate();
730:         for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
731:           const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
732:           const typename recv_overlap_type::traits::coneSequence::iterator end         = recvPatches->end();

734:           for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
735:             if (recvSection->getFiberDimension(*p_iter, *r_iter)) {
736:               newSection->updatePoint(patch, *r_iter, recvSection->restrictPoint(*p_iter, *r_iter));
737:             }
738:           }
739:         }
740:       };
743:       template<typename RecvSection>
744:       static void updateSieve(const Obj<RecvSection>& recvSection, const Obj<topology_type>& topology) {
745:         const typename RecvSection::patch_type                 patch = 0;
746:         const typename RecvSection::topology_type::sheaf_type& ranks = recvSection->getTopology()->getPatches();
747:         const Obj<typename topology_type::sieve_type>&         sieve = topology->getPatch(patch);

749:         for(typename RecvSection::topology_type::sheaf_type::const_iterator p_iter = ranks.begin(); p_iter != ranks.end(); ++p_iter) {
750:           int                                                              rank = p_iter->first;
751:           const Obj<typename RecvSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();

753:           for(typename RecvSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
754:             const typename RecvSection::value_type *points = recvSection->restrict(rank, *b_iter);
755:             int size = recvSection->getFiberDimension(rank, *b_iter);
756:             int c = 0;

758:             for(int p = 0; p < size; p++) {
759:               //sieve->addArrow(points[p], *b_iter, c++);
760:               sieve->addArrow(points[p], *b_iter, c);
761:             }
762:           }
763:         }
764:       };
767:       template<typename SendSection, typename RecvSection>
768:       static void coneCompletion(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const Obj<topology_type>& topology, const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection) {
769:         if (sendOverlap->commSize() == 1) return;
770:         typedef typename ALECompat::New::SectionCompletion<topology_type, typename sieve_type::point_type> completion;

772:         // Distribute cones
773:         const typename topology_type::patch_type               patch           = 0;
774:         const Obj<typename sieveCompletion::topology_type>     secTopology     = completion::createSendTopology(sendOverlap);
775:         const Obj<typename sieveCompletion::cone_size_section> coneSizeSection = new typename sieveCompletion::cone_size_section(secTopology, topology, topology->getPatch(patch));
776:         const Obj<typename sieveCompletion::cone_section>      coneSection     = new typename sieveCompletion::cone_section(secTopology, topology->getPatch(patch));
777:         completion::completeSection(sendOverlap, recvOverlap, coneSizeSection, coneSection, sendSection, recvSection);
778:         // Update cones
779:         updateSieve(recvSection, topology);
780:       };
783:       static void createPartitionOverlap(const Obj<topology_type>& topology, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
784:         const Obj<send_overlap_type>& topSendOverlap = topology->getSendOverlap();
785:         const Obj<recv_overlap_type>& topRecvOverlap = topology->getRecvOverlap();
786:         const Obj<typename send_overlap_type::traits::baseSequence> base = topSendOverlap->base();
787:         const Obj<typename recv_overlap_type::traits::capSequence>  cap  = topRecvOverlap->cap();

789:         if (base->empty()) {
790:           if (topology->commRank() == 0) {
791:             for(int p = 1; p < topology->commSize(); p++) {
792:               // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
793:               sendOverlap->addCone(p, p, p);
794:             }
795:           }
796:         } else {
797:           for(typename send_overlap_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
798:             const int& p = *b_iter;
799:             // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
800:             sendOverlap->addCone(p, p, p);
801:           }
802:         }
803:         if (cap->empty()) {
804:           if (topology->commRank() != 0) {
805:             // The arrow is from local partition point rank (color) on rank 0 (source) to remote partition point rank (target)
806:             recvOverlap->addCone(0, topology->commRank(), topology->commRank());
807:           }
808:         } else {
809:           for(typename recv_overlap_type::traits::capSequence::iterator c_iter = cap->begin(); c_iter != cap->end(); ++c_iter) {
810:             const int& p = *c_iter;
811:             // The arrow is from local partition point rank (color) on rank p (source) to remote partition point rank (target)
812:             recvOverlap->addCone(p, topology->commRank(), topology->commRank());
813:           }
814:         }
815:         if (topology->debug()) {
816:           sendOverlap->view("Initial send overlap");
817:           recvOverlap->view("Initial receive overlap");
818:         }
819:       }
822:       template<typename Partitioner>
823:       static typename Partitioner::part_type *createAssignment(const Obj<topology_type>& topology, const int dim, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
824:         // 1) Form partition point overlap a priori
825:         createPartitionOverlap(topology, sendOverlap, recvOverlap);
826:         if (topology->debug()) {
827:           sendOverlap->view("Send overlap for partition");
828:           recvOverlap->view("Receive overlap for partition");
829:         }
830:         // 2) Partition the mesh
831:         return Partitioner::partitionSieve(topology, dim);
832:       };
835:       // Partition a topology on process 0 and scatter to all processes
836:       static void scatterTopology(const Obj<topology_type>& topology, const int dim, const Obj<topology_type>& topologyNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const std::string& partitioner, const Obj<topology_type>& subTopology = NULL, const Obj<topology_type>& subTopologyNew = NULL) {
837:         if (partitioner == "chaco") {
838: #ifdef PETSC_HAVE_CHACO
839:           typedef typename ALECompat::New::Chaco::Partitioner<topology_type> Partitioner;
840:           typedef typename ALECompat::New::Partitioner<topology_type>        GenPartitioner;
841:           typedef typename Partitioner::part_type                      part_type;

843:           part_type *assignment = scatterTopology<Partitioner>(topology, dim, topologyNew, sendOverlap, recvOverlap);
844:           if (!subTopology.isNull() && !subTopologyNew.isNull()) {
845:             part_type *subAssignment = GenPartitioner::subordinatePartition(topology, 1, subTopology, assignment);
846:             Obj<send_overlap_type> subSendOverlap = new send_overlap_type(topology->comm(), topology->debug());
847:             Obj<recv_overlap_type> subRecvOverlap = new recv_overlap_type(topology->comm(), topology->debug());
848:             const typename topology_type::patch_type patch      = 0;
849:             const Obj<sieve_type>&                   sieve      = subTopology->getPatch(patch);
850:             const Obj<sieve_type>&                   sieveNew   = new Mesh::sieve_type(subTopology->comm(), subTopology->debug());
851:             const int                                numCells   = subTopology->heightStratum(patch, 0)->size();

853:             createPartitionOverlap(subTopology, subSendOverlap, subRecvOverlap);
854:             subTopologyNew->setPatch(0, sieveNew);
855:             sieveCompletion::scatterSieve(subTopology, sieve, dim, sieveNew, subSendOverlap, subRecvOverlap, numCells, subAssignment);
856:             subTopologyNew->stratify();
857:             subTopologyNew->setDistSendOverlap(subSendOverlap);
858:             subTopologyNew->setDistRecvOverlap(subRecvOverlap);
859:             if (subAssignment != NULL) delete [] subAssignment;
860:           }
861:           if (assignment != NULL) delete [] assignment;
862: #else
863:           throw ALE::Exception("Chaco is not installed. Reconfigure with the flag --download-chaco");
864: #endif
865:         } else if (partitioner == "parmetis") {
866: #ifdef PETSC_HAVE_PARMETIS
867:           typedef typename ALECompat::New::ParMetis::Partitioner<topology_type> Partitioner;
868:           typedef typename Partitioner::part_type                         part_type;

870:           part_type *assignment = scatterTopology<Partitioner>(topology, dim, topologyNew, sendOverlap, recvOverlap);
871:           if (assignment != NULL) delete [] assignment;
872: #else
873:           throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
874: #endif
875:         } else {
876:           throw ALE::Exception("Unknown partitioner");
877:         }
878:       };
879:       template<typename Partitioner>
880:       static typename Partitioner::part_type *scatterTopology(const Obj<topology_type>& topology, const int dim, const Obj<topology_type>& topologyNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
881:         typename Partitioner::part_type         *assignment = createAssignment<Partitioner>(topology, dim, sendOverlap, recvOverlap);
882:         const typename topology_type::patch_type patch      = 0;
883:         const Obj<sieve_type>&                   sieve      = topology->getPatch(patch);
884:         const Obj<sieve_type>&                   sieveNew   = topologyNew->getPatch(patch);
885:         const int                                numCells   = topology->heightStratum(patch, 0)->size();

887:         sieveCompletion::scatterSieve(topology, sieve, dim, sieveNew, sendOverlap, recvOverlap, numCells, assignment);
888:         topologyNew->stratify();
889:         return assignment;
890:       };
893:       static Obj<Mesh> distributeMesh(const Obj<Mesh>& serialMesh, const std::string& partitioner = "chaco") {
894:         Obj<Mesh> parallelMesh = new Mesh(serialMesh->comm(), serialMesh->getDimension(), serialMesh->debug());
895:         const Obj<Mesh::topology_type>& serialTopology   = serialMesh->getTopology();
896:         const Obj<Mesh::topology_type>& parallelTopology = new Mesh::topology_type(serialMesh->comm(), serialMesh->debug());
897:         const Obj<Mesh::topology_type>& tractionTopology = new Mesh::topology_type(serialMesh->comm(), serialMesh->debug());
898:         const Obj<Mesh::sieve_type>&    sieve            = new Mesh::sieve_type(serialMesh->comm(), serialMesh->debug());
899:         const int                       dim              = serialMesh->getDimension();

901:         //if (serialMesh->getDistributed()) return serialMesh;
902:         ALE_LOG_EVENT_BEGIN;
903:         parallelTopology->setPatch(0, sieve);
904:         parallelMesh->setTopology(parallelTopology);
905:         if (serialMesh->debug()) {
906:           serialMesh->view("Serial topology");
907:         }

909:         // Distribute cones
910:         Obj<send_overlap_type> sendOverlap = new send_overlap_type(serialTopology->comm(), serialTopology->debug());
911:         Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(serialTopology->comm(), serialTopology->debug());
912:         if (serialMesh->hasRealSection("traction")) {
913:           const Obj<Mesh::real_section_type>& traction = serialMesh->getRealSection("traction");

915:           scatterTopology(serialTopology, dim, parallelTopology, sendOverlap, recvOverlap, partitioner, traction->getTopology(), tractionTopology);
916:         } else {
917:           scatterTopology(serialTopology, dim, parallelTopology, sendOverlap, recvOverlap, partitioner);
918:         }
919:         parallelTopology->setDistSendOverlap(sendOverlap);
920:         parallelTopology->setDistRecvOverlap(recvOverlap);

922:         // Distribute labels
923:         const typename topology_type::labels_type& labels = serialTopology->getLabels();

925:         for(typename topology_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
926:           for(typename topology_type::label_type::const_iterator pl_iter = l_iter->second.begin(); pl_iter != l_iter->second.end(); ++pl_iter) {
927:             if (parallelTopology->hasLabel(l_iter->first, pl_iter->first)) continue;
928:             const Obj<typename topology_type::patch_label_type>& serialLabel   = pl_iter->second;
929:             const Obj<typename topology_type::patch_label_type>& parallelLabel = parallelTopology->createLabel(pl_iter->first, l_iter->first);
930:             // Create local label
931:             const Obj<typename topology_type::patch_label_type::traits::baseSequence>& base = serialLabel->base();
932:             const Obj<typename topology_type::sieve_type>& parallelSieve = parallelTopology->getPatch(pl_iter->first);

934:             for(typename topology_type::patch_label_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
935:               if (parallelSieve->capContains(*b_iter) || parallelSieve->baseContains(*b_iter)) {
936:                 parallelLabel->addArrow(*serialLabel->cone(*b_iter)->begin(), *b_iter);
937:               }
938:             }
939:             // Get remote labels
940:             sieveCompletion::scatterCones(serialLabel, parallelLabel, sendOverlap, recvOverlap);
941:           }
942:         }

944:         // Distribute sections
945:         Obj<std::set<std::string> > sections = serialMesh->getRealSections();

947:         for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
948:           if (*name == "traction") {
949:             parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), tractionTopology, sendOverlap, recvOverlap));
950:           } else {
951:             parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
952:           }
953:         }
954:         sections = serialMesh->getIntSections();
955:         for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
956:           parallelMesh->setIntSection(*name, distributeSection(serialMesh->getIntSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
957:         }
958:         sections = serialMesh->getPairSections();
959:         for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
960:           parallelMesh->setPairSection(*name, distributeSection(serialMesh->getPairSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
961:         }

963:         // This is necessary since we create types (like PartitionSection) on a subset of processors
964:         if (parallelMesh->debug()) {parallelMesh->view("Parallel Mesh");}
965:         parallelMesh->setDistributed(true);
966:         ALE_LOG_EVENT_END;
967:         return parallelMesh;
968:       };
971:       template<typename Section>
972:       static Obj<Section> distributeSection(const Obj<Section>& serialSection, const Obj<Mesh::topology_type>& parallelTopology, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
973:         if (serialSection->debug()) {
974:           serialSection->view("Serial Section");
975:         }
976:         typedef OverlapValues<send_overlap_type, typename sieveCompletion::topology_type, typename Section::value_type> send_section_type;
977:         typedef OverlapValues<recv_overlap_type, typename sieveCompletion::topology_type, typename Section::value_type> recv_section_type;
978:         typedef SizeSection<Section>      SectionSizer;
979:         // TEST THIS! I think this is unnecessary
980:         typedef PatchlessSection<Section> SectionFiller;
981:         Obj<Section>                       parallelSection = new Section(parallelTopology);
982:         const typename Section::patch_type patch           = 0;
983:         const Obj<send_section_type>       sendSection     = new send_section_type(serialSection->comm(), serialSection->debug());
984:         const Obj<recv_section_type>       recvSection     = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
985:         const Obj<SectionSizer>            sizer           = new SectionSizer(serialSection, patch);
986:         const Obj<SectionFiller>           filler          = new SectionFiller(serialSection, patch);

988:         updateSectionLocal(serialSection, parallelSection);
989:         sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
990:         updateSectionRemote(recvOverlap, recvSection, parallelSection);
991:         if (parallelSection->debug()) {
992:           parallelSection->view("Parallel Section");
993:         }
994:         return parallelSection;
995:       };
998:       template<typename Section>
999:       static void completeSection(const Obj<Section>& section) {
1000:         typedef typename Section::topology_type                       topology_type;
1001:         typedef typename Distribution<topology_type>::sieveCompletion sieveCompletion;
1002:         typedef typename topology_type::send_overlap_type             send_overlap_type;
1003:         typedef typename topology_type::recv_overlap_type             recv_overlap_type;
1004:         typedef typename Section::value_type                          value_type;
1005:         typedef OverlapValues<send_overlap_type, typename sieveCompletion::topology_type, value_type> send_section_type;
1006:         typedef OverlapValues<recv_overlap_type, typename sieveCompletion::topology_type, value_type> recv_section_type;
1007:         typedef SizeSection<Section>      SectionSizer;
1008:         typedef PatchlessSection<Section> SectionFiller;
1009:         const Obj<topology_type>&                topology = section->getTopology();
1010:         const typename topology_type::patch_type patch    = 0;
1011:         const int                                debug    = section->debug();
1012:         topology->constructOverlap(patch);

1014:         const Obj<send_overlap_type> sendOverlap = topology->getSendOverlap();
1015:         const Obj<recv_overlap_type> recvOverlap = topology->getRecvOverlap();
1016:         const Obj<send_section_type> sendSection = new send_section_type(section->comm(), section->debug());
1017:         const Obj<recv_section_type> recvSection = new recv_section_type(section->comm(), sendSection->getTag(), section->debug());
1018:         const Obj<SectionSizer>      sizer       = new SectionSizer(section, patch);
1019:         const Obj<SectionFiller>     filler      = new SectionFiller(section, patch);

1021:         sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
1022:         // Update section with remote data
1023:         const Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = topology->getRecvOverlap()->base();

1025:         for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
1026:           const Obj<typename recv_overlap_type::traits::coneSequence>&     recvPatches = recvOverlap->cone(*r_iter);
1027:           const typename recv_overlap_type::traits::coneSequence::iterator end         = recvPatches->end();

1029:           for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
1030:             if (recvSection->getFiberDimension(*p_iter, p_iter.color())) {
1031:               if (debug) {std::cout << "["<<section->commRank()<<"]Completed point " << *r_iter << std::endl;}
1032:               section->updateAddPoint(patch, *r_iter, recvSection->restrictPoint(*p_iter, p_iter.color()));
1033:             }
1034:           }
1035:         }
1036:       };
1037:     };
1038:   }
1039: }
1040: #endif