Actual source code: Distribution.hh
1: #ifndef included_ALE_Distribution_hh
2: #define included_ALE_Distribution_hh
4: #ifndef included_ALE_Mesh_hh
5: #include <Mesh.hh>
6: #endif
8: #ifndef included_ALE_Partitioner_hh
9: #include <Partitioner.hh>
10: #endif
12: #ifndef included_ALE_Completion_hh
13: #include <Completion.hh>
14: #endif
16: // Attempt to unify all of the distribution mechanisms:
17: // one to many (distributeMesh)
18: // many to one (unifyMesh)
19: // many to many (Numbering)
20: // as well as things being distributed
21: // Section
22: // Sieve (This sends two sections, the points and cones)
23: // Numbering (Should be an integer section)
24: // Global Order (should be an integer section with extra methods)
25: //
26: // 0) Create the new object to hold the communicated data
27: //
28: // 1) Create Overlap
29: // There may be special ways to do this based upon what we know at the time
30: //
31: // 2) Create send and receive sections over the interface
32: // These have a flat topology now, consisting only of the overlap nodes
33: // We could make a full topology on the overlap (maybe it is necessary for higher order)
34: //
35: // 3) Communication section
36: // Create sizer sections on interface (uses constant sizer)
37: // Communicate sizes on interface (uses custom filler)
38: // Fill send section
39: // sendSection->startCommunication();
40: // recvSection->startCommunication();
41: // sendSection->endCommunication();
42: // recvSection->endCommunication();
43: //
44: // Create section on interface (uses previous sizer)
45: // Communicate values on interface (uses custom filler)
46: // Same stuff as above
47: //
48: // 4) Update new section with old local values (can be done in between the communication?)
49: // Loop over patches in new topology
50: // Loop over chart from patch in old atlas
51: // If this point is in the new sieve from patch
52: // Set to old fiber dimension
53: // Order and allocate new section
54: // Repeat loop, but update values
55: //
56: // 5) Update new section with old received values
57: // Loop over patches in discrete topology of receive section (these are ranks)
58: // Loop over base of discrete sieve (we should transform this to a chart to match above)
59: // Get new patch from overlap, or should the receive patches be <rank, patch>?
60: // Guaranteed to be in the new sieve from patch (but we could check anyway)
61: // Set to recevied fiber dimension
62: // Order and allocate new section
63: // Repeat loop, but update values
64: //
65: // 6) Synchronize PETSc tags (can I get around this?)
66: namespace ALE {
67: template<typename Bundle_>
68: class Distribution {
69: public:
70: typedef Bundle_ bundle_type;
71: typedef typename bundle_type::sieve_type sieve_type;
72: typedef typename bundle_type::point_type point_type;
73: typedef typename bundle_type::alloc_type alloc_type;
74: typedef typename ALE::New::Completion<bundle_type, typename sieve_type::point_type> sieveCompletion;
75: typedef typename ALE::New::SectionCompletion<bundle_type, typename bundle_type::real_section_type::value_type> sectionCompletion;
76: typedef typename sectionCompletion::send_overlap_type send_overlap_type;
77: typedef typename sectionCompletion::recv_overlap_type recv_overlap_type;
78: public:
81: static void createPartitionOverlap(const Obj<bundle_type>& bundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
82: const Obj<send_overlap_type>& topSendOverlap = bundle->getSendOverlap();
83: const Obj<recv_overlap_type>& topRecvOverlap = bundle->getRecvOverlap();
84: const Obj<typename send_overlap_type::traits::baseSequence> base = topSendOverlap->base();
85: const Obj<typename recv_overlap_type::traits::capSequence> cap = topRecvOverlap->cap();
86: const int rank = bundle->commRank();
88: if (base->empty()) {
89: if (rank == 0) {
90: for(int p = 1; p < bundle->commSize(); p++) {
91: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
92: sendOverlap->addCone(p, p, p);
93: }
94: }
95: } else {
96: for(typename send_overlap_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
97: const int& p = *b_iter;
98: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
99: sendOverlap->addCone(p, p, p);
100: }
101: }
102: if (cap->empty()) {
103: if (rank != 0) {
104: // The arrow is from local partition point rank (color) on rank 0 (source) to remote partition point rank (target)
105: recvOverlap->addCone(0, rank, rank);
106: }
107: } else {
108: for(typename recv_overlap_type::traits::capSequence::iterator c_iter = cap->begin(); c_iter != cap->end(); ++c_iter) {
109: const int& p = *c_iter;
110: // The arrow is from local partition point rank (color) on rank p (source) to remote partition point rank (target)
111: recvOverlap->addCone(p, rank, rank);
112: }
113: }
114: };
117: template<typename Partitioner>
118: static typename Partitioner::part_type *createAssignment(const Obj<bundle_type>& bundle, const int dim, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const int height = 0) {
119: // 1) Form partition point overlap a priori
120: createPartitionOverlap(bundle, sendOverlap, recvOverlap);
121: if (bundle->debug()) {
122: sendOverlap->view("Send overlap for partition");
123: recvOverlap->view("Receive overlap for partition");
124: }
125: // 2) Partition the mesh
126: if (height == 0) {
127: return Partitioner::partitionSieve(bundle, dim);
128: } else if (height == 1) {
129: return Partitioner::partitionSieveByFace(bundle, dim);
130: }
131: throw ALE::Exception("Invalid partition height");
132: };
135: // Partition a bundle on process 0 and scatter to all processes
136: static void scatterBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const std::string& partitioner, const int height = 0, const Obj<bundle_type>& subBundle = NULL, const Obj<bundle_type>& subBundleNew = NULL) {
137: if (height == 0) {
138: if (partitioner == "chaco") {
139: #ifdef PETSC_HAVE_CHACO
140: typedef typename ALE::New::Chaco::Partitioner<bundle_type> Partitioner;
141: typedef typename ALE::New::Partitioner<bundle_type> GenPartitioner;
142: typedef typename Partitioner::part_type part_type;
144: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
145: if (!subBundle.isNull() && !subBundleNew.isNull()) {
146: part_type *subAssignment = GenPartitioner::subordinatePartition(bundle, 1, subBundle, assignment);
147: const Obj<sieve_type>& sieve = subBundle->getSieve();
148: const Obj<sieve_type>& sieveNew = new Mesh::sieve_type(subBundle->comm(), subBundle->debug());
149: const int numCells = subBundle->heightStratum(height)->size();
151: subBundleNew->setSieve(sieveNew);
152: sieveCompletion::scatterSieve(subBundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numCells, subAssignment);
153: subBundleNew->stratify();
154: if (subAssignment != NULL) delete [] subAssignment;
155: }
156: if (assignment != NULL) delete [] assignment;
157: #else
158: throw ALE::Exception("Chaco is not installed. Reconfigure with the flag --download-chaco");
159: #endif
160: } else if (partitioner == "parmetis") {
161: #ifdef PETSC_HAVE_PARMETIS
162: typedef typename ALE::New::ParMetis::Partitioner<bundle_type> Partitioner;
163: typedef typename ALE::New::Partitioner<bundle_type> GenPartitioner;
164: typedef typename Partitioner::part_type part_type;
166: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
167: if (!subBundle.isNull() && !subBundleNew.isNull()) {
168: part_type *subAssignment = GenPartitioner::subordinatePartition(bundle, 1, subBundle, assignment);
169: const Obj<sieve_type>& sieve = subBundle->getSieve();
170: const Obj<sieve_type>& sieveNew = new Mesh::sieve_type(subBundle->comm(), subBundle->debug());
171: const int numCells = subBundle->heightStratum(height)->size();
173: subBundleNew->setSieve(sieveNew);
174: sieveCompletion::scatterSieve(subBundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numCells, subAssignment);
175: subBundleNew->stratify();
176: if (subAssignment != NULL) delete [] subAssignment;
177: }
178: if (assignment != NULL) delete [] assignment;
179: #else
180: throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
181: #endif
182: } else {
183: throw ALE::Exception("Unknown partitioner");
184: }
185: } else if (height == 1) {
186: if (partitioner == "zoltan") {
187: #ifdef PETSC_HAVE_ZOLTAN
188: typedef typename ALE::New::Zoltan::Partitioner<bundle_type> Partitioner;
189: typedef typename Partitioner::part_type part_type;
191: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
192: if (assignment != NULL) delete [] assignment;
193: #else
194: throw ALE::Exception("Zoltan is not installed. Reconfigure with the flag --download-zoltan");
195: #endif
196: } else if (partitioner == "parmetis") {
197: #ifdef PETSC_HAVE_PARMETIS
198: typedef typename ALE::New::ParMetis::Partitioner<bundle_type> Partitioner;
199: typedef typename Partitioner::part_type part_type;
201: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
202: if (assignment != NULL) delete [] assignment;
203: #else
204: throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
205: #endif
206: } else {
207: throw ALE::Exception("Unknown partitioner");
208: }
209: } else {
210: throw ALE::Exception("Invalid partition height");
211: }
212: };
213: template<typename Partitioner>
214: static typename Partitioner::part_type *scatterBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const int height = 0) {
215: typename Partitioner::part_type *assignment = createAssignment<Partitioner>(bundle, dim, sendOverlap, recvOverlap, height);
216: const Obj<sieve_type>& sieve = bundle->getSieve();
217: const Obj<sieve_type>& sieveNew = bundleNew->getSieve();
218: const int numPoints = bundle->heightStratum(height)->size();
220: sieveCompletion::scatterSieve(bundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numPoints, assignment);
221: bundleNew->stratify();
222: return assignment;
223: };
226: static Obj<ALE::Mesh> distributeMesh(const Obj<ALE::Mesh>& serialMesh, const int height = 0, const std::string& partitioner = "chaco") {
227: MPI_Comm comm = serialMesh->comm();
228: const int dim = serialMesh->getDimension();
229: Obj<ALE::Mesh> parallelMesh = new ALE::Mesh(comm, dim, serialMesh->debug());
230: const Obj<ALE::Mesh::sieve_type>& parallelSieve = new ALE::Mesh::sieve_type(comm, serialMesh->debug());
232: ALE_LOG_EVENT_BEGIN;
233: parallelMesh->setSieve(parallelSieve);
234: if (serialMesh->debug()) {serialMesh->view("Serial mesh");}
236: // Distribute cones
237: Obj<send_overlap_type> sendOverlap = new send_overlap_type(comm, serialMesh->debug());
238: Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(comm, serialMesh->debug());
239: scatterBundle(serialMesh, dim, parallelMesh, sendOverlap, recvOverlap, partitioner, height);
240: parallelMesh->setDistSendOverlap(sendOverlap);
241: parallelMesh->setDistRecvOverlap(recvOverlap);
243: // Distribute labels
244: const typename bundle_type::labels_type& labels = serialMesh->getLabels();
246: for(typename bundle_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
247: if (parallelMesh->hasLabel(l_iter->first)) continue;
248: const Obj<typename bundle_type::label_type>& serialLabel = l_iter->second;
249: const Obj<typename bundle_type::label_type>& parallelLabel = parallelMesh->createLabel(l_iter->first);
250: // Create local label
251: #define NEW_LABEL
252: #ifdef NEW_LABEL
253: parallelLabel->add(serialLabel, parallelSieve);
254: #else
255: const Obj<typename bundle_type::label_type::traits::baseSequence>& base = serialLabel->base();
257: for(typename bundle_type::label_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
258: if (parallelSieve->capContains(*b_iter) || parallelSieve->baseContains(*b_iter)) {
259: parallelLabel->addArrow(*serialLabel->cone(*b_iter)->begin(), *b_iter);
260: }
261: }
262: #endif
263: // Get remote labels
264: sieveCompletion::scatterCones(serialLabel, parallelLabel, sendOverlap, recvOverlap);
265: }
267: // Distribute sections
268: Obj<std::set<std::string> > sections = serialMesh->getRealSections();
270: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
271: parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), parallelMesh, sendOverlap, recvOverlap));
272: }
273: sections = serialMesh->getIntSections();
274: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
275: parallelMesh->setIntSection(*name, distributeSection(serialMesh->getIntSection(*name), parallelMesh, sendOverlap, recvOverlap));
276: }
277: sections = serialMesh->getArrowSections();
279: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
280: parallelMesh->setArrowSection(*name, distributeArrowSection(serialMesh->getArrowSection(*name), serialMesh, parallelMesh, sendOverlap, recvOverlap));
281: }
282: if (parallelMesh->debug()) {parallelMesh->view("Parallel Mesh");}
283: ALE_LOG_EVENT_END;
284: return parallelMesh;
285: };
288: template<typename Section>
289: static void updateSectionLocal(const Obj<Section>& oldSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
290: const Obj<typename bundle_type::sieve_type>& newSieve = newBundle->getSieve();
291: const typename Section::atlas_type::chart_type& oldChart = oldSection->getChart();
293: for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
294: if (newSieve->capContains(*c_iter) || newSieve->baseContains(*c_iter)) {
295: newSection->setFiberDimension(*c_iter, oldSection->getFiberDimension(*c_iter));
296: }
297: }
298: newBundle->allocate(newSection);
299: const typename Section::atlas_type::chart_type& newChart = newSection->getChart();
301: for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
302: newSection->updatePointAll(*c_iter, oldSection->restrictPoint(*c_iter));
303: }
304: };
307: template<typename RecvSection, typename Section>
308: static void updateSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
309: Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();
311: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
312: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
313: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
315: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
316: newSection->addPoint(*r_iter, recvSection->getSection(*p_iter)->getFiberDimension(*r_iter));
317: }
318: }
319: newBundle->reallocate(newSection);
320: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
321: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
322: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
324: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
325: if (recvSection->getSection(*p_iter)->getFiberDimension(*r_iter)) {
326: newSection->updatePointAll(*r_iter, recvSection->getSection(*p_iter)->restrictPoint(*r_iter));
327: }
328: }
329: }
330: };
333: template<typename Section>
334: static Obj<Section> distributeSection(const Obj<Section>& serialSection, const Obj<bundle_type>& parallelBundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
335: if (serialSection->debug()) {
336: serialSection->view("Serial Section");
337: }
338: typedef typename alloc_type::template rebind<typename Section::value_type>::other value_alloc_type;
339: typedef ALE::Field<send_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > send_section_type;
340: typedef ALE::Field<recv_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > recv_section_type;
341: typedef ALE::New::SizeSection<Section> SectionSizer;
342: Obj<Section> parallelSection = new Section(serialSection->comm(), serialSection->debug());
343: const Obj<send_section_type> sendSection = new send_section_type(serialSection->comm(), serialSection->debug());
344: const Obj<recv_section_type> recvSection = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
345: const Obj<SectionSizer> sizer = new SectionSizer(serialSection);
347: updateSectionLocal(serialSection, parallelBundle, parallelSection);
348: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, serialSection, sendSection, recvSection);
349: updateSectionRemote(recvOverlap, recvSection, parallelBundle, parallelSection);
350: if (parallelSection->debug()) {
351: parallelSection->view("Parallel Section");
352: }
353: return parallelSection;
354: };
357: template<typename Section>
358: static void updateArrowSectionLocal(const Obj<Section>& oldSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
359: const Obj<typename bundle_type::sieve_type>& newSieve = newBundle->getSieve();
360: const typename Section::atlas_type::chart_type& oldChart = oldSection->getChart();
362: for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
363: // Dmitry should provide a Sieve::contains(MinimalArrow) method
364: if (newSieve->capContains(c_iter->source) && newSieve->baseContains(c_iter->target)) {
365: newSection->setFiberDimension(*c_iter, oldSection->getFiberDimension(*c_iter));
366: }
367: }
368: //newBundle->allocate(newSection);
369: const typename Section::atlas_type::chart_type& newChart = newSection->getChart();
371: for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
372: newSection->updatePointAll(*c_iter, oldSection->restrictPoint(*c_iter));
373: }
374: };
377: template<typename RecvSection, typename Section>
378: static void updateArrowSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
379: Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();
381: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
382: const Obj<typename bundle_type::sieve_type::traits::coneSequence>& cone = newBundle->getSieve()->cone(*r_iter);
383: const typename bundle_type::sieve_type::traits::coneSequence::iterator end = cone->end();
385: for(typename bundle_type::sieve_type::traits::coneSequence::iterator c_iter = cone->begin(); c_iter != end; ++c_iter) {
386: newSection->setFiberDimension(typename Section::point_type(*c_iter, *r_iter), 1);
387: }
388: }
389: //newBundle->reallocate(newSection);
390: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
391: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
392: const typename recv_overlap_type::traits::coneSequence::iterator recvEnd = recvPatches->end();
394: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != recvEnd; ++p_iter) {
395: const Obj<typename RecvSection::section_type>& section = recvSection->getSection(*p_iter);
397: if (section->getFiberDimension(*r_iter)) {
398: const Obj<typename bundle_type::sieve_type::traits::coneSequence>& cone = newBundle->getSieve()->cone(*r_iter);
399: const typename bundle_type::sieve_type::traits::coneSequence::iterator end = cone->end();
400: const typename RecvSection::value_type *values = section->restrictPoint(*r_iter);
401: int c = -1;
403: for(typename bundle_type::sieve_type::traits::coneSequence::iterator c_iter = cone->begin(); c_iter != end; ++c_iter) {
404: newSection->updatePoint(typename Section::point_type(*c_iter, *r_iter), &values[++c]);
405: }
406: }
407: }
408: }
409: };
412: template<typename Section>
413: static Obj<Section> distributeArrowSection(const Obj<Section>& serialSection, const Obj<bundle_type>& serialBundle, const Obj<bundle_type>& parallelBundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
414: if (serialSection->debug()) {
415: serialSection->view("Serial ArrowSection");
416: }
417: typedef typename alloc_type::template rebind<typename Section::value_type>::other value_alloc_type;
418: typedef ALE::Field<send_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > send_section_type;
419: typedef ALE::Field<recv_overlap_type, int, ALE::Section<point_type, typename Section::value_type, value_alloc_type> > recv_section_type;
420: typedef ALE::New::ConeSizeSection<bundle_type, sieve_type> SectionSizer;
421: typedef ALE::New::ArrowSection<sieve_type, Section> ArrowFiller;
422: Obj<Section> parallelSection = new Section(serialSection->comm(), serialSection->debug());
423: const Obj<send_section_type> sendSection = new send_section_type(serialSection->comm(), serialSection->debug());
424: const Obj<recv_section_type> recvSection = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
425: const Obj<SectionSizer> sizer = new SectionSizer(serialBundle, serialBundle->getSieve());
426: const Obj<ArrowFiller> filler = new ArrowFiller(serialBundle->getSieve(), serialSection);
428: updateArrowSectionLocal(serialSection, parallelBundle, parallelSection);
429: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
430: updateArrowSectionRemote(recvOverlap, recvSection, parallelBundle, parallelSection);
431: if (parallelSection->debug()) {
432: parallelSection->view("Parallel ArrowSection");
433: }
434: return parallelSection;
435: };
436: static void unifyBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
437: typedef int part_type;
438: const Obj<sieve_type>& sieve = bundle->getSieve();
439: const Obj<sieve_type>& sieveNew = bundleNew->getSieve();
440: const int rank = bundle->commRank();
441: const int debug = bundle->debug();
443: // 1) Form partition point overlap a priori
444: if (rank == 0) {
445: for(int p = 1; p < sieve->commSize(); p++) {
446: // The arrow is from remote partition point 0 on rank p to local partition point 0
447: recvOverlap->addCone(p, 0, 0);
448: }
449: } else {
450: // The arrow is from local partition point 0 to remote partition point 0 on rank 0
451: sendOverlap->addCone(0, 0, 0);
452: }
453: if (debug) {
454: sendOverlap->view("Send overlap for partition");
455: recvOverlap->view("Receive overlap for partition");
456: }
457: // 2) Partition the mesh
458: int numCells = bundle->heightStratum(0)->size();
459: part_type *assignment = new part_type[numCells];
461: for(int c = 0; c < numCells; ++c) {
462: assignment[c] = 0;
463: }
464: // 3) Scatter the sieve
465: sieveCompletion::scatterSieve(bundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, 0, numCells, assignment);
466: bundleNew->stratify();
467: // 4) Cleanup
468: if (assignment != NULL) delete [] assignment;
469: };
472: static Obj<ALE::Mesh> unifyMesh(const Obj<ALE::Mesh>& parallelMesh) {
473: const int dim = parallelMesh->getDimension();
474: Obj<ALE::Mesh> serialMesh = new ALE::Mesh(parallelMesh->comm(), dim, parallelMesh->debug());
475: const Obj<ALE::Mesh::sieve_type>& serialSieve = new ALE::Mesh::sieve_type(parallelMesh->comm(), parallelMesh->debug());
477: ALE_LOG_EVENT_BEGIN;
478: serialMesh->setSieve(serialSieve);
479: if (parallelMesh->debug()) {
480: parallelMesh->view("Parallel topology");
481: }
483: // Unify cones
484: Obj<send_overlap_type> sendOverlap = new send_overlap_type(serialMesh->comm(), serialMesh->debug());
485: Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(serialMesh->comm(), serialMesh->debug());
486: unifyBundle(parallelMesh, dim, serialMesh, sendOverlap, recvOverlap);
487: serialMesh->setDistSendOverlap(sendOverlap);
488: serialMesh->setDistRecvOverlap(recvOverlap);
490: // Unify labels
491: const typename bundle_type::labels_type& labels = parallelMesh->getLabels();
493: for(typename bundle_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
494: if (serialMesh->hasLabel(l_iter->first)) continue;
495: const Obj<typename bundle_type::label_type>& parallelLabel = l_iter->second;
496: const Obj<typename bundle_type::label_type>& serialLabel = serialMesh->createLabel(l_iter->first);
498: sieveCompletion::scatterCones(parallelLabel, serialLabel, sendOverlap, recvOverlap);
499: }
501: // Unify coordinates
502: Obj<std::set<std::string> > sections = parallelMesh->getRealSections();
504: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
505: serialMesh->setRealSection(*name, distributeSection(parallelMesh->getRealSection(*name), serialMesh, sendOverlap, recvOverlap));
506: }
507: sections = parallelMesh->getIntSections();
508: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
509: serialMesh->setIntSection(*name, distributeSection(parallelMesh->getIntSection(*name), serialMesh, sendOverlap, recvOverlap));
510: }
511: sections = parallelMesh->getArrowSections();
512: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
513: serialMesh->setArrowSection(*name, distributeArrowSection(parallelMesh->getArrowSection(*name), parallelMesh, serialMesh, sendOverlap, recvOverlap));
514: }
515: if (serialMesh->debug()) {serialMesh->view("Serial Mesh");}
516: ALE_LOG_EVENT_END;
517: return serialMesh;
518: };
519: public: // Do not like these
522: // This is just crappy. We could introduce another phase to find out exactly what
523: // indices people do not have in the global order after communication
524: template<typename SendSection, typename RecvSection>
525: static void updateOverlap(const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
526: const typename SendSection::sheaf_type& sendRanks = sendSection->getPatches();
527: const typename RecvSection::sheaf_type& recvRanks = recvSection->getPatches();
529: for(typename SendSection::sheaf_type::const_iterator p_iter = sendRanks.begin(); p_iter != sendRanks.end(); ++p_iter) {
530: const typename SendSection::patch_type& rank = p_iter->first;
531: const Obj<typename SendSection::section_type>& section = p_iter->second;
532: const typename SendSection::section_type::chart_type& chart = section->getChart();
534: for(typename SendSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
535: const typename SendSection::value_type *points = section->restrictPoint(*b_iter);
536: const int size = section->getFiberDimension(*b_iter);
538: for(int p = 0; p < size; p++) {
539: sendOverlap->addArrow(points[p], rank, points[p]);
540: }
541: }
542: }
543: for(typename RecvSection::sheaf_type::const_iterator p_iter = recvRanks.begin(); p_iter != recvRanks.end(); ++p_iter) {
544: const typename RecvSection::patch_type& rank = p_iter->first;
545: const Obj<typename RecvSection::section_type>& section = p_iter->second;
546: const typename RecvSection::section_type::chart_type& chart = section->getChart();
548: for(typename RecvSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
549: const typename RecvSection::value_type *points = section->restrictPoint(*b_iter);
550: const int size = section->getFiberDimension(*b_iter);
552: for(int p = 0; p < size; p++) {
553: recvOverlap->addArrow(rank, points[p], points[p]);
554: }
555: }
556: }
557: };
560: template<typename RecvSection>
561: static void updateSieve(const Obj<RecvSection>& recvSection, const Obj<sieve_type>& sieve) {
562: const typename RecvSection::sheaf_type& ranks = recvSection->getPatches();
564: for(typename RecvSection::sheaf_type::const_iterator p_iter = ranks.begin(); p_iter != ranks.end(); ++p_iter) {
565: const Obj<typename RecvSection::section_type>& section = p_iter->second;
566: const typename RecvSection::section_type::chart_type& chart = section->getChart();
568: for(typename RecvSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
569: const typename RecvSection::value_type *points = section->restrictPoint(*b_iter);
570: int size = section->getFiberDimension(*b_iter);
571: int c = 0;
573: for(int p = 0; p < size; p++) {
574: //sieve->addArrow(points[p], *b_iter, c++);
575: sieve->addArrow(points[p], *b_iter, c);
576: }
577: }
578: }
579: };
582: template<typename SendSection, typename RecvSection>
583: static void coneCompletion(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const Obj<bundle_type>& bundle, const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection) {
584: if (sendOverlap->commSize() == 1) return;
585: // Distribute cones
586: const Obj<sieve_type>& sieve = bundle->getSieve();
587: const Obj<typename sieveCompletion::topology_type> secTopology = sieveCompletion::completion::createSendTopology(sendOverlap);
588: const Obj<typename sieveCompletion::cone_size_section> coneSizeSection = new typename sieveCompletion::cone_size_section(bundle, sieve);
589: const Obj<typename sieveCompletion::cone_section> coneSection = new typename sieveCompletion::cone_section(sieve);
590: sieveCompletion::completion::completeSection(sendOverlap, recvOverlap, coneSizeSection, coneSection, sendSection, recvSection);
591: // Update cones
592: updateSieve(recvSection, sieve);
593: };
596: template<typename Section>
597: static void completeSection(const Obj<bundle_type>& bundle, const Obj<Section>& section) {
598: typedef typename Distribution<bundle_type>::sieveCompletion sieveCompletion;
599: typedef typename bundle_type::send_overlap_type send_overlap_type;
600: typedef typename bundle_type::recv_overlap_type recv_overlap_type;
601: typedef typename Section::value_type value_type;
602: typedef typename alloc_type::template rebind<typename Section::value_type>::other value_alloc_type;
603: typedef typename ALE::Field<send_overlap_type, int, ALE::Section<point_type, value_type, value_alloc_type> > send_section_type;
604: typedef typename ALE::Field<recv_overlap_type, int, ALE::Section<point_type, value_type, value_alloc_type> > recv_section_type;
605: typedef ALE::New::SizeSection<Section> SectionSizer;
606: const int debug = section->debug();
608: bundle->constructOverlap();
609: const Obj<send_overlap_type> sendOverlap = bundle->getSendOverlap();
610: const Obj<recv_overlap_type> recvOverlap = bundle->getRecvOverlap();
611: const Obj<send_section_type> sendSection = new send_section_type(section->comm(), section->debug());
612: const Obj<recv_section_type> recvSection = new recv_section_type(section->comm(), sendSection->getTag(), section->debug());
613: const Obj<SectionSizer> sizer = new SectionSizer(section);
615: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, section, sendSection, recvSection);
616: // Update section with remote data
617: const Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = bundle->getRecvOverlap()->base();
619: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
620: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
621: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
623: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
624: if (recvSection->getSection(*p_iter)->getFiberDimension(p_iter.color())) {
625: if (debug) {std::cout << "["<<section->commRank()<<"]Completed point " << *r_iter << std::endl;}
626: section->updateAddPoint(*r_iter, recvSection->getSection(*p_iter)->restrictPoint(p_iter.color()));
627: }
628: }
629: }
630: };
631: };
632: }
634: namespace ALECompat {
635: namespace New {
636: template<typename Topology_>
637: class Distribution {
638: public:
639: typedef Topology_ topology_type;
640: typedef typename topology_type::sieve_type sieve_type;
641: typedef ALECompat::New::Completion<Topology_, Mesh::sieve_type::point_type> sieveCompletion;
642: typedef ALECompat::New::SectionCompletion<Topology_, Mesh::real_section_type::value_type> sectionCompletion;
643: typedef typename sectionCompletion::send_overlap_type send_overlap_type;
644: typedef typename sectionCompletion::recv_overlap_type recv_overlap_type;
645: public:
648: // This is just crappy. WE could introduce another phase to find out exactly what
649: // indices people do not have in the global order after communication
650: template<typename SendSection, typename RecvSection>
651: static void updateOverlap(const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
652: const typename SendSection::topology_type::sheaf_type& sendRanks = sendSection->getTopology()->getPatches();
653: const typename RecvSection::topology_type::sheaf_type& recvRanks = recvSection->getTopology()->getPatches();
655: for(typename SendSection::topology_type::sheaf_type::const_iterator p_iter = sendRanks.begin(); p_iter != sendRanks.end(); ++p_iter) {
656: int rank = p_iter->first;
657: const Obj<typename SendSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();
659: for(typename SendSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
660: const typename SendSection::value_type *points = sendSection->restrict(rank, *b_iter);
661: int size = sendSection->getFiberDimension(rank, *b_iter);
663: for(int p = 0; p < size; p++) {
664: sendOverlap->addArrow(points[p], rank, points[p]);
665: }
666: }
667: }
668: for(typename RecvSection::topology_type::sheaf_type::const_iterator p_iter = recvRanks.begin(); p_iter != recvRanks.end(); ++p_iter) {
669: int rank = p_iter->first;
670: const Obj<typename RecvSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();
672: for(typename RecvSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
673: const typename RecvSection::value_type *points = recvSection->restrict(rank, *b_iter);
674: int size = recvSection->getFiberDimension(rank, *b_iter);
676: for(int p = 0; p < size; p++) {
677: recvOverlap->addArrow(rank, points[p], points[p]);
678: }
679: }
680: }
681: };
682: static void createLabelOverlap(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
683: };
686: template<typename Section>
687: static void updateSectionLocal(const Obj<Section>& oldSection, const Obj<Section>& newSection)
688: {
689: const typename Section::topology_type::sheaf_type& patches = newSection->getTopology()->getPatches();
691: for(typename Section::topology_type::sheaf_type::const_iterator p_iter = patches.begin(); p_iter != patches.end(); ++p_iter) {
692: const typename Section::patch_type& patch = p_iter->first;
693: const Obj<typename Section::topology_type::sieve_type>& newSieve = p_iter->second;
694: if (!oldSection->hasPatch(patch)) continue;
695: const typename Section::atlas_type::chart_type& oldChart = oldSection->getPatch(patch);
697: for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
698: if (newSieve->hasPoint(*c_iter)) {
699: newSection->setFiberDimension(patch, *c_iter, oldSection->getFiberDimension(patch, *c_iter));
700: }
701: }
702: }
703: newSection->allocate();
704: for(typename Section::topology_type::sheaf_type::const_iterator p_iter = patches.begin(); p_iter != patches.end(); ++p_iter) {
705: const typename Section::patch_type& patch = p_iter->first;
706: if (!oldSection->hasPatch(patch)) continue;
707: const typename Section::atlas_type::chart_type& newChart = newSection->getPatch(patch);
709: for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
710: newSection->updatePoint(patch, *c_iter, oldSection->restrictPoint(patch, *c_iter));
711: }
712: }
713: };
716: template<typename RecvSection, typename Section>
717: static void updateSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<Section>& newSection) {
718: const Mesh::real_section_type::patch_type patch = 0;
719: Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();
721: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
722: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
723: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
725: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
726: newSection->addPoint(patch, *r_iter, recvSection->getFiberDimension(*p_iter, *r_iter));
727: }
728: }
729: newSection->reallocate();
730: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
731: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
732: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
734: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
735: if (recvSection->getFiberDimension(*p_iter, *r_iter)) {
736: newSection->updatePoint(patch, *r_iter, recvSection->restrictPoint(*p_iter, *r_iter));
737: }
738: }
739: }
740: };
743: template<typename RecvSection>
744: static void updateSieve(const Obj<RecvSection>& recvSection, const Obj<topology_type>& topology) {
745: const typename RecvSection::patch_type patch = 0;
746: const typename RecvSection::topology_type::sheaf_type& ranks = recvSection->getTopology()->getPatches();
747: const Obj<typename topology_type::sieve_type>& sieve = topology->getPatch(patch);
749: for(typename RecvSection::topology_type::sheaf_type::const_iterator p_iter = ranks.begin(); p_iter != ranks.end(); ++p_iter) {
750: int rank = p_iter->first;
751: const Obj<typename RecvSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();
753: for(typename RecvSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
754: const typename RecvSection::value_type *points = recvSection->restrict(rank, *b_iter);
755: int size = recvSection->getFiberDimension(rank, *b_iter);
756: int c = 0;
758: for(int p = 0; p < size; p++) {
759: //sieve->addArrow(points[p], *b_iter, c++);
760: sieve->addArrow(points[p], *b_iter, c);
761: }
762: }
763: }
764: };
767: template<typename SendSection, typename RecvSection>
768: static void coneCompletion(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const Obj<topology_type>& topology, const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection) {
769: if (sendOverlap->commSize() == 1) return;
770: typedef typename ALECompat::New::SectionCompletion<topology_type, typename sieve_type::point_type> completion;
772: // Distribute cones
773: const typename topology_type::patch_type patch = 0;
774: const Obj<typename sieveCompletion::topology_type> secTopology = completion::createSendTopology(sendOverlap);
775: const Obj<typename sieveCompletion::cone_size_section> coneSizeSection = new typename sieveCompletion::cone_size_section(secTopology, topology, topology->getPatch(patch));
776: const Obj<typename sieveCompletion::cone_section> coneSection = new typename sieveCompletion::cone_section(secTopology, topology->getPatch(patch));
777: completion::completeSection(sendOverlap, recvOverlap, coneSizeSection, coneSection, sendSection, recvSection);
778: // Update cones
779: updateSieve(recvSection, topology);
780: };
783: static void createPartitionOverlap(const Obj<topology_type>& topology, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
784: const Obj<send_overlap_type>& topSendOverlap = topology->getSendOverlap();
785: const Obj<recv_overlap_type>& topRecvOverlap = topology->getRecvOverlap();
786: const Obj<typename send_overlap_type::traits::baseSequence> base = topSendOverlap->base();
787: const Obj<typename recv_overlap_type::traits::capSequence> cap = topRecvOverlap->cap();
789: if (base->empty()) {
790: if (topology->commRank() == 0) {
791: for(int p = 1; p < topology->commSize(); p++) {
792: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
793: sendOverlap->addCone(p, p, p);
794: }
795: }
796: } else {
797: for(typename send_overlap_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
798: const int& p = *b_iter;
799: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
800: sendOverlap->addCone(p, p, p);
801: }
802: }
803: if (cap->empty()) {
804: if (topology->commRank() != 0) {
805: // The arrow is from local partition point rank (color) on rank 0 (source) to remote partition point rank (target)
806: recvOverlap->addCone(0, topology->commRank(), topology->commRank());
807: }
808: } else {
809: for(typename recv_overlap_type::traits::capSequence::iterator c_iter = cap->begin(); c_iter != cap->end(); ++c_iter) {
810: const int& p = *c_iter;
811: // The arrow is from local partition point rank (color) on rank p (source) to remote partition point rank (target)
812: recvOverlap->addCone(p, topology->commRank(), topology->commRank());
813: }
814: }
815: if (topology->debug()) {
816: sendOverlap->view("Initial send overlap");
817: recvOverlap->view("Initial receive overlap");
818: }
819: }
822: template<typename Partitioner>
823: static typename Partitioner::part_type *createAssignment(const Obj<topology_type>& topology, const int dim, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
824: // 1) Form partition point overlap a priori
825: createPartitionOverlap(topology, sendOverlap, recvOverlap);
826: if (topology->debug()) {
827: sendOverlap->view("Send overlap for partition");
828: recvOverlap->view("Receive overlap for partition");
829: }
830: // 2) Partition the mesh
831: return Partitioner::partitionSieve(topology, dim);
832: };
835: // Partition a topology on process 0 and scatter to all processes
836: static void scatterTopology(const Obj<topology_type>& topology, const int dim, const Obj<topology_type>& topologyNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const std::string& partitioner, const Obj<topology_type>& subTopology = NULL, const Obj<topology_type>& subTopologyNew = NULL) {
837: if (partitioner == "chaco") {
838: #ifdef PETSC_HAVE_CHACO
839: typedef typename ALECompat::New::Chaco::Partitioner<topology_type> Partitioner;
840: typedef typename ALECompat::New::Partitioner<topology_type> GenPartitioner;
841: typedef typename Partitioner::part_type part_type;
843: part_type *assignment = scatterTopology<Partitioner>(topology, dim, topologyNew, sendOverlap, recvOverlap);
844: if (!subTopology.isNull() && !subTopologyNew.isNull()) {
845: part_type *subAssignment = GenPartitioner::subordinatePartition(topology, 1, subTopology, assignment);
846: Obj<send_overlap_type> subSendOverlap = new send_overlap_type(topology->comm(), topology->debug());
847: Obj<recv_overlap_type> subRecvOverlap = new recv_overlap_type(topology->comm(), topology->debug());
848: const typename topology_type::patch_type patch = 0;
849: const Obj<sieve_type>& sieve = subTopology->getPatch(patch);
850: const Obj<sieve_type>& sieveNew = new Mesh::sieve_type(subTopology->comm(), subTopology->debug());
851: const int numCells = subTopology->heightStratum(patch, 0)->size();
853: createPartitionOverlap(subTopology, subSendOverlap, subRecvOverlap);
854: subTopologyNew->setPatch(0, sieveNew);
855: sieveCompletion::scatterSieve(subTopology, sieve, dim, sieveNew, subSendOverlap, subRecvOverlap, numCells, subAssignment);
856: subTopologyNew->stratify();
857: subTopologyNew->setDistSendOverlap(subSendOverlap);
858: subTopologyNew->setDistRecvOverlap(subRecvOverlap);
859: if (subAssignment != NULL) delete [] subAssignment;
860: }
861: if (assignment != NULL) delete [] assignment;
862: #else
863: throw ALE::Exception("Chaco is not installed. Reconfigure with the flag --download-chaco");
864: #endif
865: } else if (partitioner == "parmetis") {
866: #ifdef PETSC_HAVE_PARMETIS
867: typedef typename ALECompat::New::ParMetis::Partitioner<topology_type> Partitioner;
868: typedef typename Partitioner::part_type part_type;
870: part_type *assignment = scatterTopology<Partitioner>(topology, dim, topologyNew, sendOverlap, recvOverlap);
871: if (assignment != NULL) delete [] assignment;
872: #else
873: throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
874: #endif
875: } else {
876: throw ALE::Exception("Unknown partitioner");
877: }
878: };
879: template<typename Partitioner>
880: static typename Partitioner::part_type *scatterTopology(const Obj<topology_type>& topology, const int dim, const Obj<topology_type>& topologyNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
881: typename Partitioner::part_type *assignment = createAssignment<Partitioner>(topology, dim, sendOverlap, recvOverlap);
882: const typename topology_type::patch_type patch = 0;
883: const Obj<sieve_type>& sieve = topology->getPatch(patch);
884: const Obj<sieve_type>& sieveNew = topologyNew->getPatch(patch);
885: const int numCells = topology->heightStratum(patch, 0)->size();
887: sieveCompletion::scatterSieve(topology, sieve, dim, sieveNew, sendOverlap, recvOverlap, numCells, assignment);
888: topologyNew->stratify();
889: return assignment;
890: };
893: static Obj<Mesh> distributeMesh(const Obj<Mesh>& serialMesh, const std::string& partitioner = "chaco") {
894: Obj<Mesh> parallelMesh = new Mesh(serialMesh->comm(), serialMesh->getDimension(), serialMesh->debug());
895: const Obj<Mesh::topology_type>& serialTopology = serialMesh->getTopology();
896: const Obj<Mesh::topology_type>& parallelTopology = new Mesh::topology_type(serialMesh->comm(), serialMesh->debug());
897: const Obj<Mesh::topology_type>& tractionTopology = new Mesh::topology_type(serialMesh->comm(), serialMesh->debug());
898: const Obj<Mesh::sieve_type>& sieve = new Mesh::sieve_type(serialMesh->comm(), serialMesh->debug());
899: const int dim = serialMesh->getDimension();
901: //if (serialMesh->getDistributed()) return serialMesh;
902: ALE_LOG_EVENT_BEGIN;
903: parallelTopology->setPatch(0, sieve);
904: parallelMesh->setTopology(parallelTopology);
905: if (serialMesh->debug()) {
906: serialMesh->view("Serial topology");
907: }
909: // Distribute cones
910: Obj<send_overlap_type> sendOverlap = new send_overlap_type(serialTopology->comm(), serialTopology->debug());
911: Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(serialTopology->comm(), serialTopology->debug());
912: if (serialMesh->hasRealSection("traction")) {
913: const Obj<Mesh::real_section_type>& traction = serialMesh->getRealSection("traction");
915: scatterTopology(serialTopology, dim, parallelTopology, sendOverlap, recvOverlap, partitioner, traction->getTopology(), tractionTopology);
916: } else {
917: scatterTopology(serialTopology, dim, parallelTopology, sendOverlap, recvOverlap, partitioner);
918: }
919: parallelTopology->setDistSendOverlap(sendOverlap);
920: parallelTopology->setDistRecvOverlap(recvOverlap);
922: // Distribute labels
923: const typename topology_type::labels_type& labels = serialTopology->getLabels();
925: for(typename topology_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
926: for(typename topology_type::label_type::const_iterator pl_iter = l_iter->second.begin(); pl_iter != l_iter->second.end(); ++pl_iter) {
927: if (parallelTopology->hasLabel(l_iter->first, pl_iter->first)) continue;
928: const Obj<typename topology_type::patch_label_type>& serialLabel = pl_iter->second;
929: const Obj<typename topology_type::patch_label_type>& parallelLabel = parallelTopology->createLabel(pl_iter->first, l_iter->first);
930: // Create local label
931: const Obj<typename topology_type::patch_label_type::traits::baseSequence>& base = serialLabel->base();
932: const Obj<typename topology_type::sieve_type>& parallelSieve = parallelTopology->getPatch(pl_iter->first);
934: for(typename topology_type::patch_label_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
935: if (parallelSieve->capContains(*b_iter) || parallelSieve->baseContains(*b_iter)) {
936: parallelLabel->addArrow(*serialLabel->cone(*b_iter)->begin(), *b_iter);
937: }
938: }
939: // Get remote labels
940: sieveCompletion::scatterCones(serialLabel, parallelLabel, sendOverlap, recvOverlap);
941: }
942: }
944: // Distribute sections
945: Obj<std::set<std::string> > sections = serialMesh->getRealSections();
947: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
948: if (*name == "traction") {
949: parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), tractionTopology, sendOverlap, recvOverlap));
950: } else {
951: parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
952: }
953: }
954: sections = serialMesh->getIntSections();
955: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
956: parallelMesh->setIntSection(*name, distributeSection(serialMesh->getIntSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
957: }
958: sections = serialMesh->getPairSections();
959: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
960: parallelMesh->setPairSection(*name, distributeSection(serialMesh->getPairSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
961: }
963: // This is necessary since we create types (like PartitionSection) on a subset of processors
964: if (parallelMesh->debug()) {parallelMesh->view("Parallel Mesh");}
965: parallelMesh->setDistributed(true);
966: ALE_LOG_EVENT_END;
967: return parallelMesh;
968: };
971: template<typename Section>
972: static Obj<Section> distributeSection(const Obj<Section>& serialSection, const Obj<Mesh::topology_type>& parallelTopology, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
973: if (serialSection->debug()) {
974: serialSection->view("Serial Section");
975: }
976: typedef OverlapValues<send_overlap_type, typename sieveCompletion::topology_type, typename Section::value_type> send_section_type;
977: typedef OverlapValues<recv_overlap_type, typename sieveCompletion::topology_type, typename Section::value_type> recv_section_type;
978: typedef SizeSection<Section> SectionSizer;
979: // TEST THIS! I think this is unnecessary
980: typedef PatchlessSection<Section> SectionFiller;
981: Obj<Section> parallelSection = new Section(parallelTopology);
982: const typename Section::patch_type patch = 0;
983: const Obj<send_section_type> sendSection = new send_section_type(serialSection->comm(), serialSection->debug());
984: const Obj<recv_section_type> recvSection = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
985: const Obj<SectionSizer> sizer = new SectionSizer(serialSection, patch);
986: const Obj<SectionFiller> filler = new SectionFiller(serialSection, patch);
988: updateSectionLocal(serialSection, parallelSection);
989: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
990: updateSectionRemote(recvOverlap, recvSection, parallelSection);
991: if (parallelSection->debug()) {
992: parallelSection->view("Parallel Section");
993: }
994: return parallelSection;
995: };
998: template<typename Section>
999: static void completeSection(const Obj<Section>& section) {
1000: typedef typename Section::topology_type topology_type;
1001: typedef typename Distribution<topology_type>::sieveCompletion sieveCompletion;
1002: typedef typename topology_type::send_overlap_type send_overlap_type;
1003: typedef typename topology_type::recv_overlap_type recv_overlap_type;
1004: typedef typename Section::value_type value_type;
1005: typedef OverlapValues<send_overlap_type, typename sieveCompletion::topology_type, value_type> send_section_type;
1006: typedef OverlapValues<recv_overlap_type, typename sieveCompletion::topology_type, value_type> recv_section_type;
1007: typedef SizeSection<Section> SectionSizer;
1008: typedef PatchlessSection<Section> SectionFiller;
1009: const Obj<topology_type>& topology = section->getTopology();
1010: const typename topology_type::patch_type patch = 0;
1011: const int debug = section->debug();
1012: topology->constructOverlap(patch);
1014: const Obj<send_overlap_type> sendOverlap = topology->getSendOverlap();
1015: const Obj<recv_overlap_type> recvOverlap = topology->getRecvOverlap();
1016: const Obj<send_section_type> sendSection = new send_section_type(section->comm(), section->debug());
1017: const Obj<recv_section_type> recvSection = new recv_section_type(section->comm(), sendSection->getTag(), section->debug());
1018: const Obj<SectionSizer> sizer = new SectionSizer(section, patch);
1019: const Obj<SectionFiller> filler = new SectionFiller(section, patch);
1021: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
1022: // Update section with remote data
1023: const Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = topology->getRecvOverlap()->base();
1025: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
1026: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
1027: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
1029: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
1030: if (recvSection->getFiberDimension(*p_iter, p_iter.color())) {
1031: if (debug) {std::cout << "["<<section->commRank()<<"]Completed point " << *r_iter << std::endl;}
1032: section->updateAddPoint(patch, *r_iter, recvSection->restrictPoint(*p_iter, p_iter.color()));
1033: }
1034: }
1035: }
1036: };
1037: };
1038: }
1039: }
1040: #endif