Actual source code: Distribution.hh
1: #ifndef included_ALE_Distribution_hh
2: #define included_ALE_Distribution_hh
4: #ifndef included_ALE_Mesh_hh
5: #include <Mesh.hh>
6: #endif
8: #ifndef included_ALE_Partitioner_hh
9: #include <Partitioner.hh>
10: #endif
12: #ifndef included_ALE_Completion_hh
13: #include <Completion.hh>
14: #endif
16: // Attempt to unify all of the distribution mechanisms:
17: // one to many (distributeMesh)
18: // many to one (unifyMesh)
19: // many to many (Numbering)
20: // as well as things being distributed
21: // Section
22: // Sieve (This sends two sections, the points and cones)
23: // Numbering (Should be an integer section)
24: // Global Order (should be an integer section with extra methods)
25: //
26: // 0) Create the new object to hold the communicated data
27: //
28: // 1) Create Overlap
29: // There may be special ways to do this based upon what we know at the time
30: //
31: // 2) Create send and receive sections over the interface
32: // These have a flat topology now, consisting only of the overlap nodes
33: // We could make a full topology on the overlap (maybe it is necessary for higher order)
34: //
35: // 3) Communication section
36: // Create sizer sections on interface (uses constant sizer)
37: // Communicate sizes on interface (uses custom filler)
38: // Fill send section
39: // sendSection->startCommunication();
40: // recvSection->startCommunication();
41: // sendSection->endCommunication();
42: // recvSection->endCommunication();
43: //
44: // Create section on interface (uses previous sizer)
45: // Communicate values on interface (uses custom filler)
46: // Same stuff as above
47: //
48: // 4) Update new section with old local values (can be done in between the communication?)
49: // Loop over patches in new topology
50: // Loop over chart from patch in old atlas
51: // If this point is in the new sieve from patch
52: // Set to old fiber dimension
53: // Order and allocate new section
54: // Repeat loop, but update values
55: //
56: // 5) Update new section with old received values
57: // Loop over patches in discrete topology of receive section (these are ranks)
58: // Loop over base of discrete sieve (we should transform this to a chart to match above)
59: // Get new patch from overlap, or should the receive patches be <rank, patch>?
60: // Guaranteed to be in the new sieve from patch (but we could check anyway)
61: // Set to recevied fiber dimension
62: // Order and allocate new section
63: // Repeat loop, but update values
64: //
65: // 6) Synchronize PETSc tags (can I get around this?)
66: namespace ALE {
67: template<typename Bundle_>
68: class Distribution {
69: public:
70: typedef Bundle_ bundle_type;
71: typedef typename bundle_type::sieve_type sieve_type;
72: typedef typename bundle_type::point_type point_type;
73: typedef typename ALE::New::Completion<bundle_type, typename sieve_type::point_type> sieveCompletion;
74: typedef typename ALE::New::SectionCompletion<bundle_type, typename bundle_type::real_section_type::value_type> sectionCompletion;
75: typedef typename sectionCompletion::send_overlap_type send_overlap_type;
76: typedef typename sectionCompletion::recv_overlap_type recv_overlap_type;
77: public:
80: static void createPartitionOverlap(const Obj<bundle_type>& bundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
81: const Obj<send_overlap_type>& topSendOverlap = bundle->getSendOverlap();
82: const Obj<recv_overlap_type>& topRecvOverlap = bundle->getRecvOverlap();
83: const Obj<typename send_overlap_type::traits::baseSequence> base = topSendOverlap->base();
84: const Obj<typename recv_overlap_type::traits::capSequence> cap = topRecvOverlap->cap();
85: const int rank = bundle->commRank();
87: if (base->empty()) {
88: if (rank == 0) {
89: for(int p = 1; p < bundle->commSize(); p++) {
90: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
91: sendOverlap->addCone(p, p, p);
92: }
93: }
94: } else {
95: for(typename send_overlap_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
96: const int& p = *b_iter;
97: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
98: sendOverlap->addCone(p, p, p);
99: }
100: }
101: if (cap->empty()) {
102: if (rank != 0) {
103: // The arrow is from local partition point rank (color) on rank 0 (source) to remote partition point rank (target)
104: recvOverlap->addCone(0, rank, rank);
105: }
106: } else {
107: for(typename recv_overlap_type::traits::capSequence::iterator c_iter = cap->begin(); c_iter != cap->end(); ++c_iter) {
108: const int& p = *c_iter;
109: // The arrow is from local partition point rank (color) on rank p (source) to remote partition point rank (target)
110: recvOverlap->addCone(p, rank, rank);
111: }
112: }
113: };
116: template<typename Partitioner>
117: static typename Partitioner::part_type *createAssignment(const Obj<bundle_type>& bundle, const int dim, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const int height = 0) {
118: // 1) Form partition point overlap a priori
119: createPartitionOverlap(bundle, sendOverlap, recvOverlap);
120: if (bundle->debug()) {
121: sendOverlap->view("Send overlap for partition");
122: recvOverlap->view("Receive overlap for partition");
123: }
124: // 2) Partition the mesh
125: if (height == 0) {
126: return Partitioner::partitionSieve(bundle, dim);
127: } else if (height == 1) {
128: return Partitioner::partitionSieveByFace(bundle, dim);
129: }
130: throw ALE::Exception("Invalid partition height");
131: };
134: // Partition a bundle on process 0 and scatter to all processes
135: static void scatterBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const std::string& partitioner, const int height = 0, const Obj<bundle_type>& subBundle = NULL, const Obj<bundle_type>& subBundleNew = NULL) {
136: if (height == 0) {
137: if (partitioner == "chaco") {
138: #ifdef PETSC_HAVE_CHACO
139: typedef typename ALE::New::Chaco::Partitioner<bundle_type> Partitioner;
140: typedef typename ALE::New::Partitioner<bundle_type> GenPartitioner;
141: typedef typename Partitioner::part_type part_type;
143: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
144: if (!subBundle.isNull() && !subBundleNew.isNull()) {
145: part_type *subAssignment = GenPartitioner::subordinatePartition(bundle, 1, subBundle, assignment);
146: const Obj<sieve_type>& sieve = subBundle->getSieve();
147: const Obj<sieve_type>& sieveNew = new Mesh::sieve_type(subBundle->comm(), subBundle->debug());
148: const int numCells = subBundle->heightStratum(height)->size();
150: subBundleNew->setSieve(sieveNew);
151: sieveCompletion::scatterSieve(subBundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numCells, subAssignment);
152: subBundleNew->stratify();
153: if (subAssignment != NULL) delete [] subAssignment;
154: }
155: if (assignment != NULL) delete [] assignment;
156: #else
157: throw ALE::Exception("Chaco is not installed. Reconfigure with the flag --download-chaco");
158: #endif
159: } else if (partitioner == "parmetis") {
160: #ifdef PETSC_HAVE_PARMETIS
161: typedef typename ALE::New::ParMetis::Partitioner<bundle_type> Partitioner;
162: typedef typename ALE::New::Partitioner<bundle_type> GenPartitioner;
163: typedef typename Partitioner::part_type part_type;
165: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
166: if (!subBundle.isNull() && !subBundleNew.isNull()) {
167: part_type *subAssignment = GenPartitioner::subordinatePartition(bundle, 1, subBundle, assignment);
168: const Obj<sieve_type>& sieve = subBundle->getSieve();
169: const Obj<sieve_type>& sieveNew = new Mesh::sieve_type(subBundle->comm(), subBundle->debug());
170: const int numCells = subBundle->heightStratum(height)->size();
172: subBundleNew->setSieve(sieveNew);
173: sieveCompletion::scatterSieve(subBundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numCells, subAssignment);
174: subBundleNew->stratify();
175: if (subAssignment != NULL) delete [] subAssignment;
176: }
177: if (assignment != NULL) delete [] assignment;
178: #else
179: throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
180: #endif
181: } else {
182: throw ALE::Exception("Unknown partitioner");
183: }
184: } else if (height == 1) {
185: if (partitioner == "zoltan") {
186: #ifdef PETSC_HAVE_ZOLTAN
187: typedef typename ALE::New::Zoltan::Partitioner<bundle_type> Partitioner;
188: typedef typename Partitioner::part_type part_type;
190: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
191: if (assignment != NULL) delete [] assignment;
192: #else
193: throw ALE::Exception("Zoltan is not installed. Reconfigure with the flag --download-zoltan");
194: #endif
195: } else if (partitioner == "parmetis") {
196: #ifdef PETSC_HAVE_PARMETIS
197: typedef typename ALE::New::ParMetis::Partitioner<bundle_type> Partitioner;
198: typedef typename Partitioner::part_type part_type;
200: part_type *assignment = scatterBundle<Partitioner>(bundle, dim, bundleNew, sendOverlap, recvOverlap, height);
201: if (assignment != NULL) delete [] assignment;
202: #else
203: throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
204: #endif
205: } else {
206: throw ALE::Exception("Unknown partitioner");
207: }
208: } else {
209: throw ALE::Exception("Invalid partition height");
210: }
211: };
212: template<typename Partitioner>
213: static typename Partitioner::part_type *scatterBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const int height = 0) {
214: typename Partitioner::part_type *assignment = createAssignment<Partitioner>(bundle, dim, sendOverlap, recvOverlap, height);
215: const Obj<sieve_type>& sieve = bundle->getSieve();
216: const Obj<sieve_type>& sieveNew = bundleNew->getSieve();
217: const int numPoints = bundle->heightStratum(height)->size();
219: sieveCompletion::scatterSieve(bundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, height, numPoints, assignment);
220: bundleNew->stratify();
221: return assignment;
222: };
225: static Obj<ALE::Mesh> distributeMesh(const Obj<ALE::Mesh>& serialMesh, const int height = 0, const std::string& partitioner = "chaco") {
226: MPI_Comm comm = serialMesh->comm();
227: const int dim = serialMesh->getDimension();
228: Obj<ALE::Mesh> parallelMesh = new ALE::Mesh(comm, dim, serialMesh->debug());
229: const Obj<ALE::Mesh::sieve_type>& parallelSieve = new ALE::Mesh::sieve_type(comm, serialMesh->debug());
231: ALE_LOG_EVENT_BEGIN;
232: parallelMesh->setSieve(parallelSieve);
233: if (serialMesh->debug()) {serialMesh->view("Serial mesh");}
235: // Distribute cones
236: Obj<send_overlap_type> sendOverlap = new send_overlap_type(comm, serialMesh->debug());
237: Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(comm, serialMesh->debug());
238: scatterBundle(serialMesh, dim, parallelMesh, sendOverlap, recvOverlap, partitioner, height);
239: parallelMesh->setDistSendOverlap(sendOverlap);
240: parallelMesh->setDistRecvOverlap(recvOverlap);
242: // Distribute labels
243: const typename bundle_type::labels_type& labels = serialMesh->getLabels();
245: for(typename bundle_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
246: if (parallelMesh->hasLabel(l_iter->first)) continue;
247: const Obj<typename bundle_type::label_type>& serialLabel = l_iter->second;
248: const Obj<typename bundle_type::label_type>& parallelLabel = parallelMesh->createLabel(l_iter->first);
249: // Create local label
250: #define NEW_LABEL
251: #ifdef NEW_LABEL
252: parallelLabel->add(serialLabel, parallelSieve);
253: #else
254: const Obj<typename bundle_type::label_type::traits::baseSequence>& base = serialLabel->base();
256: for(typename bundle_type::label_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
257: if (parallelSieve->capContains(*b_iter) || parallelSieve->baseContains(*b_iter)) {
258: parallelLabel->addArrow(*serialLabel->cone(*b_iter)->begin(), *b_iter);
259: }
260: }
261: #endif
262: // Get remote labels
263: sieveCompletion::scatterCones(serialLabel, parallelLabel, sendOverlap, recvOverlap);
264: }
266: // Distribute sections
267: Obj<std::set<std::string> > sections = serialMesh->getRealSections();
269: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
270: parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), parallelMesh, sendOverlap, recvOverlap));
271: }
272: sections = serialMesh->getIntSections();
273: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
274: parallelMesh->setIntSection(*name, distributeSection(serialMesh->getIntSection(*name), parallelMesh, sendOverlap, recvOverlap));
275: }
276: sections = serialMesh->getArrowSections();
278: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
279: parallelMesh->setArrowSection(*name, distributeArrowSection(serialMesh->getArrowSection(*name), serialMesh, parallelMesh, sendOverlap, recvOverlap));
280: }
281: if (parallelMesh->debug()) {parallelMesh->view("Parallel Mesh");}
282: ALE_LOG_EVENT_END;
283: return parallelMesh;
284: };
287: template<typename Section>
288: static void updateSectionLocal(const Obj<Section>& oldSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
289: const Obj<typename bundle_type::sieve_type>& newSieve = newBundle->getSieve();
290: const typename Section::atlas_type::chart_type& oldChart = oldSection->getChart();
292: for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
293: if (newSieve->capContains(*c_iter) || newSieve->baseContains(*c_iter)) {
294: newSection->setFiberDimension(*c_iter, oldSection->getFiberDimension(*c_iter));
295: }
296: }
297: newBundle->allocate(newSection);
298: const typename Section::atlas_type::chart_type& newChart = newSection->getChart();
300: for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
301: newSection->updatePointAll(*c_iter, oldSection->restrictPoint(*c_iter));
302: }
303: };
306: template<typename RecvSection, typename Section>
307: static void updateSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
308: Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();
310: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
311: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
312: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
314: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
315: newSection->addPoint(*r_iter, recvSection->getSection(*p_iter)->getFiberDimension(*r_iter));
316: }
317: }
318: newBundle->reallocate(newSection);
319: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
320: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
321: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
323: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
324: if (recvSection->getSection(*p_iter)->getFiberDimension(*r_iter)) {
325: newSection->updatePointAll(*r_iter, recvSection->getSection(*p_iter)->restrictPoint(*r_iter));
326: }
327: }
328: }
329: };
332: template<typename Section>
333: static Obj<Section> distributeSection(const Obj<Section>& serialSection, const Obj<bundle_type>& parallelBundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
334: if (serialSection->debug()) {
335: serialSection->view("Serial Section");
336: }
337: typedef ALE::Field<send_overlap_type, int, ALE::Section<point_type, typename Section::value_type> > send_section_type;
338: typedef ALE::Field<recv_overlap_type, int, ALE::Section<point_type, typename Section::value_type> > recv_section_type;
339: typedef ALE::New::SizeSection<Section> SectionSizer;
340: Obj<Section> parallelSection = new Section(serialSection->comm(), serialSection->debug());
341: const Obj<send_section_type> sendSection = new send_section_type(serialSection->comm(), serialSection->debug());
342: const Obj<recv_section_type> recvSection = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
343: const Obj<SectionSizer> sizer = new SectionSizer(serialSection);
345: updateSectionLocal(serialSection, parallelBundle, parallelSection);
346: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, serialSection, sendSection, recvSection);
347: updateSectionRemote(recvOverlap, recvSection, parallelBundle, parallelSection);
348: if (parallelSection->debug()) {
349: parallelSection->view("Parallel Section");
350: }
351: return parallelSection;
352: };
355: template<typename Section>
356: static void updateArrowSectionLocal(const Obj<Section>& oldSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
357: const Obj<typename bundle_type::sieve_type>& newSieve = newBundle->getSieve();
358: const typename Section::atlas_type::chart_type& oldChart = oldSection->getChart();
360: for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
361: // Dmitry should provide a Sieve::contains(MinimalArrow) method
362: if (newSieve->capContains(c_iter->source) && newSieve->baseContains(c_iter->target)) {
363: newSection->setFiberDimension(*c_iter, oldSection->getFiberDimension(*c_iter));
364: }
365: }
366: //newBundle->allocate(newSection);
367: const typename Section::atlas_type::chart_type& newChart = newSection->getChart();
369: for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
370: newSection->updatePointAll(*c_iter, oldSection->restrictPoint(*c_iter));
371: }
372: };
375: template<typename RecvSection, typename Section>
376: static void updateArrowSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<bundle_type>& newBundle, const Obj<Section>& newSection) {
377: Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();
379: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
380: const Obj<typename bundle_type::sieve_type::traits::coneSequence>& cone = newBundle->getSieve()->cone(*r_iter);
381: const typename bundle_type::sieve_type::traits::coneSequence::iterator end = cone->end();
383: for(typename bundle_type::sieve_type::traits::coneSequence::iterator c_iter = cone->begin(); c_iter != end; ++c_iter) {
384: newSection->setFiberDimension(typename Section::point_type(*c_iter, *r_iter), 1);
385: }
386: }
387: //newBundle->reallocate(newSection);
388: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
389: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
390: const typename recv_overlap_type::traits::coneSequence::iterator recvEnd = recvPatches->end();
392: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != recvEnd; ++p_iter) {
393: const Obj<typename RecvSection::section_type>& section = recvSection->getSection(*p_iter);
395: if (section->getFiberDimension(*r_iter)) {
396: const Obj<typename bundle_type::sieve_type::traits::coneSequence>& cone = newBundle->getSieve()->cone(*r_iter);
397: const typename bundle_type::sieve_type::traits::coneSequence::iterator end = cone->end();
398: const typename RecvSection::value_type *values = section->restrictPoint(*r_iter);
399: int c = -1;
401: for(typename bundle_type::sieve_type::traits::coneSequence::iterator c_iter = cone->begin(); c_iter != end; ++c_iter) {
402: newSection->updatePoint(typename Section::point_type(*c_iter, *r_iter), &values[++c]);
403: }
404: }
405: }
406: }
407: };
410: template<typename Section>
411: static Obj<Section> distributeArrowSection(const Obj<Section>& serialSection, const Obj<bundle_type>& serialBundle, const Obj<bundle_type>& parallelBundle, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
412: if (serialSection->debug()) {
413: serialSection->view("Serial ArrowSection");
414: }
415: typedef ALE::Field<send_overlap_type, int, ALE::Section<point_type, typename Section::value_type> > send_section_type;
416: typedef ALE::Field<recv_overlap_type, int, ALE::Section<point_type, typename Section::value_type> > recv_section_type;
417: typedef ALE::New::ConeSizeSection<bundle_type, sieve_type> SectionSizer;
418: typedef ALE::New::ArrowSection<sieve_type, Section> ArrowFiller;
419: Obj<Section> parallelSection = new Section(serialSection->comm(), serialSection->debug());
420: const Obj<send_section_type> sendSection = new send_section_type(serialSection->comm(), serialSection->debug());
421: const Obj<recv_section_type> recvSection = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
422: const Obj<SectionSizer> sizer = new SectionSizer(serialBundle, serialBundle->getSieve());
423: const Obj<ArrowFiller> filler = new ArrowFiller(serialBundle->getSieve(), serialSection);
425: updateArrowSectionLocal(serialSection, parallelBundle, parallelSection);
426: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
427: updateArrowSectionRemote(recvOverlap, recvSection, parallelBundle, parallelSection);
428: if (parallelSection->debug()) {
429: parallelSection->view("Parallel ArrowSection");
430: }
431: return parallelSection;
432: };
433: static void unifyBundle(const Obj<bundle_type>& bundle, const int dim, const Obj<bundle_type>& bundleNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
434: typedef int part_type;
435: const Obj<sieve_type>& sieve = bundle->getSieve();
436: const Obj<sieve_type>& sieveNew = bundleNew->getSieve();
437: const int rank = bundle->commRank();
438: const int debug = bundle->debug();
440: // 1) Form partition point overlap a priori
441: if (rank == 0) {
442: for(int p = 1; p < sieve->commSize(); p++) {
443: // The arrow is from remote partition point 0 on rank p to local partition point 0
444: recvOverlap->addCone(p, 0, 0);
445: }
446: } else {
447: // The arrow is from local partition point 0 to remote partition point 0 on rank 0
448: sendOverlap->addCone(0, 0, 0);
449: }
450: if (debug) {
451: sendOverlap->view("Send overlap for partition");
452: recvOverlap->view("Receive overlap for partition");
453: }
454: // 2) Partition the mesh
455: int numCells = bundle->heightStratum(0)->size();
456: part_type *assignment = new part_type[numCells];
458: for(int c = 0; c < numCells; ++c) {
459: assignment[c] = 0;
460: }
461: // 3) Scatter the sieve
462: sieveCompletion::scatterSieve(bundle, sieve, dim, sieveNew, sendOverlap, recvOverlap, 0, numCells, assignment);
463: bundleNew->stratify();
464: // 4) Cleanup
465: if (assignment != NULL) delete [] assignment;
466: };
469: static Obj<ALE::Mesh> unifyMesh(const Obj<ALE::Mesh>& parallelMesh) {
470: const int dim = parallelMesh->getDimension();
471: Obj<ALE::Mesh> serialMesh = new ALE::Mesh(parallelMesh->comm(), dim, parallelMesh->debug());
472: const Obj<ALE::Mesh::sieve_type>& serialSieve = new ALE::Mesh::sieve_type(parallelMesh->comm(), parallelMesh->debug());
474: ALE_LOG_EVENT_BEGIN;
475: serialMesh->setSieve(serialSieve);
476: if (parallelMesh->debug()) {
477: parallelMesh->view("Parallel topology");
478: }
480: // Unify cones
481: Obj<send_overlap_type> sendOverlap = new send_overlap_type(serialMesh->comm(), serialMesh->debug());
482: Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(serialMesh->comm(), serialMesh->debug());
483: unifyBundle(parallelMesh, dim, serialMesh, sendOverlap, recvOverlap);
484: serialMesh->setDistSendOverlap(sendOverlap);
485: serialMesh->setDistRecvOverlap(recvOverlap);
487: // Unify labels
488: const typename bundle_type::labels_type& labels = parallelMesh->getLabels();
490: for(typename bundle_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
491: if (serialMesh->hasLabel(l_iter->first)) continue;
492: const Obj<typename bundle_type::label_type>& parallelLabel = l_iter->second;
493: const Obj<typename bundle_type::label_type>& serialLabel = serialMesh->createLabel(l_iter->first);
495: sieveCompletion::scatterCones(parallelLabel, serialLabel, sendOverlap, recvOverlap);
496: }
498: // Unify coordinates
499: Obj<std::set<std::string> > sections = parallelMesh->getRealSections();
501: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
502: serialMesh->setRealSection(*name, distributeSection(parallelMesh->getRealSection(*name), serialMesh, sendOverlap, recvOverlap));
503: }
504: sections = parallelMesh->getIntSections();
505: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
506: serialMesh->setIntSection(*name, distributeSection(parallelMesh->getIntSection(*name), serialMesh, sendOverlap, recvOverlap));
507: }
508: sections = parallelMesh->getArrowSections();
509: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
510: serialMesh->setArrowSection(*name, distributeArrowSection(parallelMesh->getArrowSection(*name), parallelMesh, serialMesh, sendOverlap, recvOverlap));
511: }
512: if (serialMesh->debug()) {serialMesh->view("Serial Mesh");}
513: ALE_LOG_EVENT_END;
514: return serialMesh;
515: };
516: public: // Do not like these
519: // This is just crappy. We could introduce another phase to find out exactly what
520: // indices people do not have in the global order after communication
521: template<typename SendSection, typename RecvSection>
522: static void updateOverlap(const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
523: const typename SendSection::sheaf_type& sendRanks = sendSection->getPatches();
524: const typename RecvSection::sheaf_type& recvRanks = recvSection->getPatches();
526: for(typename SendSection::sheaf_type::const_iterator p_iter = sendRanks.begin(); p_iter != sendRanks.end(); ++p_iter) {
527: const typename SendSection::patch_type& rank = p_iter->first;
528: const Obj<typename SendSection::section_type>& section = p_iter->second;
529: const typename SendSection::section_type::chart_type& chart = section->getChart();
531: for(typename SendSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
532: const typename SendSection::value_type *points = section->restrictPoint(*b_iter);
533: const int size = section->getFiberDimension(*b_iter);
535: for(int p = 0; p < size; p++) {
536: sendOverlap->addArrow(points[p], rank, points[p]);
537: }
538: }
539: }
540: for(typename RecvSection::sheaf_type::const_iterator p_iter = recvRanks.begin(); p_iter != recvRanks.end(); ++p_iter) {
541: const typename RecvSection::patch_type& rank = p_iter->first;
542: const Obj<typename RecvSection::section_type>& section = p_iter->second;
543: const typename RecvSection::section_type::chart_type& chart = section->getChart();
545: for(typename RecvSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
546: const typename RecvSection::value_type *points = section->restrictPoint(*b_iter);
547: const int size = section->getFiberDimension(*b_iter);
549: for(int p = 0; p < size; p++) {
550: recvOverlap->addArrow(rank, points[p], points[p]);
551: }
552: }
553: }
554: };
557: template<typename RecvSection>
558: static void updateSieve(const Obj<RecvSection>& recvSection, const Obj<sieve_type>& sieve) {
559: const typename RecvSection::sheaf_type& ranks = recvSection->getPatches();
561: for(typename RecvSection::sheaf_type::const_iterator p_iter = ranks.begin(); p_iter != ranks.end(); ++p_iter) {
562: const Obj<typename RecvSection::section_type>& section = p_iter->second;
563: const typename RecvSection::section_type::chart_type& chart = section->getChart();
565: for(typename RecvSection::section_type::chart_type::iterator b_iter = chart.begin(); b_iter != chart.end(); ++b_iter) {
566: const typename RecvSection::value_type *points = section->restrictPoint(*b_iter);
567: int size = section->getFiberDimension(*b_iter);
568: int c = 0;
570: for(int p = 0; p < size; p++) {
571: //sieve->addArrow(points[p], *b_iter, c++);
572: sieve->addArrow(points[p], *b_iter, c);
573: }
574: }
575: }
576: };
579: template<typename SendSection, typename RecvSection>
580: static void coneCompletion(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const Obj<bundle_type>& bundle, const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection) {
581: if (sendOverlap->commSize() == 1) return;
582: // Distribute cones
583: const Obj<sieve_type>& sieve = bundle->getSieve();
584: const Obj<typename sieveCompletion::topology_type> secTopology = sieveCompletion::completion::createSendTopology(sendOverlap);
585: const Obj<typename sieveCompletion::cone_size_section> coneSizeSection = new typename sieveCompletion::cone_size_section(bundle, sieve);
586: const Obj<typename sieveCompletion::cone_section> coneSection = new typename sieveCompletion::cone_section(sieve);
587: sieveCompletion::completion::completeSection(sendOverlap, recvOverlap, coneSizeSection, coneSection, sendSection, recvSection);
588: // Update cones
589: updateSieve(recvSection, sieve);
590: };
593: template<typename Section>
594: static void completeSection(const Obj<bundle_type>& bundle, const Obj<Section>& section) {
595: typedef typename Distribution<bundle_type>::sieveCompletion sieveCompletion;
596: typedef typename bundle_type::send_overlap_type send_overlap_type;
597: typedef typename bundle_type::recv_overlap_type recv_overlap_type;
598: typedef typename Section::value_type value_type;
599: typedef typename ALE::Field<send_overlap_type, int, ALE::Section<point_type, value_type> > send_section_type;
600: typedef typename ALE::Field<recv_overlap_type, int, ALE::Section<point_type, value_type> > recv_section_type;
601: typedef ALE::New::SizeSection<Section> SectionSizer;
602: const int debug = section->debug();
604: bundle->constructOverlap();
605: const Obj<send_overlap_type> sendOverlap = bundle->getSendOverlap();
606: const Obj<recv_overlap_type> recvOverlap = bundle->getRecvOverlap();
607: const Obj<send_section_type> sendSection = new send_section_type(section->comm(), section->debug());
608: const Obj<recv_section_type> recvSection = new recv_section_type(section->comm(), sendSection->getTag(), section->debug());
609: const Obj<SectionSizer> sizer = new SectionSizer(section);
611: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, section, sendSection, recvSection);
612: // Update section with remote data
613: const Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = bundle->getRecvOverlap()->base();
615: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
616: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
617: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
619: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
620: if (recvSection->getSection(*p_iter)->getFiberDimension(p_iter.color())) {
621: if (debug) {std::cout << "["<<section->commRank()<<"]Completed point " << *r_iter << std::endl;}
622: section->updateAddPoint(*r_iter, recvSection->getSection(*p_iter)->restrictPoint(p_iter.color()));
623: }
624: }
625: }
626: };
627: };
628: }
630: namespace ALECompat {
631: namespace New {
632: template<typename Topology_>
633: class Distribution {
634: public:
635: typedef Topology_ topology_type;
636: typedef typename topology_type::sieve_type sieve_type;
637: typedef ALECompat::New::Completion<Topology_, Mesh::sieve_type::point_type> sieveCompletion;
638: typedef ALECompat::New::SectionCompletion<Topology_, Mesh::real_section_type::value_type> sectionCompletion;
639: typedef typename sectionCompletion::send_overlap_type send_overlap_type;
640: typedef typename sectionCompletion::recv_overlap_type recv_overlap_type;
641: public:
644: // This is just crappy. WE could introduce another phase to find out exactly what
645: // indices people do not have in the global order after communication
646: template<typename SendSection, typename RecvSection>
647: static void updateOverlap(const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
648: const typename SendSection::topology_type::sheaf_type& sendRanks = sendSection->getTopology()->getPatches();
649: const typename RecvSection::topology_type::sheaf_type& recvRanks = recvSection->getTopology()->getPatches();
651: for(typename SendSection::topology_type::sheaf_type::const_iterator p_iter = sendRanks.begin(); p_iter != sendRanks.end(); ++p_iter) {
652: int rank = p_iter->first;
653: const Obj<typename SendSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();
655: for(typename SendSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
656: const typename SendSection::value_type *points = sendSection->restrict(rank, *b_iter);
657: int size = sendSection->getFiberDimension(rank, *b_iter);
659: for(int p = 0; p < size; p++) {
660: sendOverlap->addArrow(points[p], rank, points[p]);
661: }
662: }
663: }
664: for(typename RecvSection::topology_type::sheaf_type::const_iterator p_iter = recvRanks.begin(); p_iter != recvRanks.end(); ++p_iter) {
665: int rank = p_iter->first;
666: const Obj<typename RecvSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();
668: for(typename RecvSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
669: const typename RecvSection::value_type *points = recvSection->restrict(rank, *b_iter);
670: int size = recvSection->getFiberDimension(rank, *b_iter);
672: for(int p = 0; p < size; p++) {
673: recvOverlap->addArrow(rank, points[p], points[p]);
674: }
675: }
676: }
677: };
678: static void createLabelOverlap(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
679: };
682: template<typename Section>
683: static void updateSectionLocal(const Obj<Section>& oldSection, const Obj<Section>& newSection)
684: {
685: const typename Section::topology_type::sheaf_type& patches = newSection->getTopology()->getPatches();
687: for(typename Section::topology_type::sheaf_type::const_iterator p_iter = patches.begin(); p_iter != patches.end(); ++p_iter) {
688: const typename Section::patch_type& patch = p_iter->first;
689: const Obj<typename Section::topology_type::sieve_type>& newSieve = p_iter->second;
690: if (!oldSection->hasPatch(patch)) continue;
691: const typename Section::atlas_type::chart_type& oldChart = oldSection->getPatch(patch);
693: for(typename Section::atlas_type::chart_type::const_iterator c_iter = oldChart.begin(); c_iter != oldChart.end(); ++c_iter) {
694: if (newSieve->hasPoint(*c_iter)) {
695: newSection->setFiberDimension(patch, *c_iter, oldSection->getFiberDimension(patch, *c_iter));
696: }
697: }
698: }
699: newSection->allocate();
700: for(typename Section::topology_type::sheaf_type::const_iterator p_iter = patches.begin(); p_iter != patches.end(); ++p_iter) {
701: const typename Section::patch_type& patch = p_iter->first;
702: if (!oldSection->hasPatch(patch)) continue;
703: const typename Section::atlas_type::chart_type& newChart = newSection->getPatch(patch);
705: for(typename Section::atlas_type::chart_type::const_iterator c_iter = newChart.begin(); c_iter != newChart.end(); ++c_iter) {
706: newSection->updatePoint(patch, *c_iter, oldSection->restrictPoint(patch, *c_iter));
707: }
708: }
709: };
712: template<typename RecvSection, typename Section>
713: static void updateSectionRemote(const Obj<recv_overlap_type>& recvOverlap, const Obj<RecvSection>& recvSection, const Obj<Section>& newSection) {
714: const Mesh::real_section_type::patch_type patch = 0;
715: Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = recvOverlap->base();
717: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
718: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
719: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
721: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
722: newSection->addPoint(patch, *r_iter, recvSection->getFiberDimension(*p_iter, *r_iter));
723: }
724: }
725: newSection->reallocate();
726: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
727: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
728: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
730: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
731: if (recvSection->getFiberDimension(*p_iter, *r_iter)) {
732: newSection->updatePoint(patch, *r_iter, recvSection->restrictPoint(*p_iter, *r_iter));
733: }
734: }
735: }
736: };
739: template<typename RecvSection>
740: static void updateSieve(const Obj<RecvSection>& recvSection, const Obj<topology_type>& topology) {
741: const typename RecvSection::patch_type patch = 0;
742: const typename RecvSection::topology_type::sheaf_type& ranks = recvSection->getTopology()->getPatches();
743: const Obj<typename topology_type::sieve_type>& sieve = topology->getPatch(patch);
745: for(typename RecvSection::topology_type::sheaf_type::const_iterator p_iter = ranks.begin(); p_iter != ranks.end(); ++p_iter) {
746: int rank = p_iter->first;
747: const Obj<typename RecvSection::topology_type::sieve_type::baseSequence>& base = p_iter->second->base();
749: for(typename RecvSection::topology_type::sieve_type::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
750: const typename RecvSection::value_type *points = recvSection->restrict(rank, *b_iter);
751: int size = recvSection->getFiberDimension(rank, *b_iter);
752: int c = 0;
754: for(int p = 0; p < size; p++) {
755: //sieve->addArrow(points[p], *b_iter, c++);
756: sieve->addArrow(points[p], *b_iter, c);
757: }
758: }
759: }
760: };
763: template<typename SendSection, typename RecvSection>
764: static void coneCompletion(const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const Obj<topology_type>& topology, const Obj<SendSection>& sendSection, const Obj<RecvSection>& recvSection) {
765: if (sendOverlap->commSize() == 1) return;
766: typedef typename ALECompat::New::SectionCompletion<topology_type, typename sieve_type::point_type> completion;
768: // Distribute cones
769: const typename topology_type::patch_type patch = 0;
770: const Obj<typename sieveCompletion::topology_type> secTopology = completion::createSendTopology(sendOverlap);
771: const Obj<typename sieveCompletion::cone_size_section> coneSizeSection = new typename sieveCompletion::cone_size_section(secTopology, topology, topology->getPatch(patch));
772: const Obj<typename sieveCompletion::cone_section> coneSection = new typename sieveCompletion::cone_section(secTopology, topology->getPatch(patch));
773: completion::completeSection(sendOverlap, recvOverlap, coneSizeSection, coneSection, sendSection, recvSection);
774: // Update cones
775: updateSieve(recvSection, topology);
776: };
779: static void createPartitionOverlap(const Obj<topology_type>& topology, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
780: const Obj<send_overlap_type>& topSendOverlap = topology->getSendOverlap();
781: const Obj<recv_overlap_type>& topRecvOverlap = topology->getRecvOverlap();
782: const Obj<typename send_overlap_type::traits::baseSequence> base = topSendOverlap->base();
783: const Obj<typename recv_overlap_type::traits::capSequence> cap = topRecvOverlap->cap();
785: if (base->empty()) {
786: if (topology->commRank() == 0) {
787: for(int p = 1; p < topology->commSize(); p++) {
788: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
789: sendOverlap->addCone(p, p, p);
790: }
791: }
792: } else {
793: for(typename send_overlap_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
794: const int& p = *b_iter;
795: // The arrow is from local partition point p (source) to remote partition point p (color) on rank p (target)
796: sendOverlap->addCone(p, p, p);
797: }
798: }
799: if (cap->empty()) {
800: if (topology->commRank() != 0) {
801: // The arrow is from local partition point rank (color) on rank 0 (source) to remote partition point rank (target)
802: recvOverlap->addCone(0, topology->commRank(), topology->commRank());
803: }
804: } else {
805: for(typename recv_overlap_type::traits::capSequence::iterator c_iter = cap->begin(); c_iter != cap->end(); ++c_iter) {
806: const int& p = *c_iter;
807: // The arrow is from local partition point rank (color) on rank p (source) to remote partition point rank (target)
808: recvOverlap->addCone(p, topology->commRank(), topology->commRank());
809: }
810: }
811: if (topology->debug()) {
812: sendOverlap->view("Initial send overlap");
813: recvOverlap->view("Initial receive overlap");
814: }
815: }
818: template<typename Partitioner>
819: static typename Partitioner::part_type *createAssignment(const Obj<topology_type>& topology, const int dim, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
820: // 1) Form partition point overlap a priori
821: createPartitionOverlap(topology, sendOverlap, recvOverlap);
822: if (topology->debug()) {
823: sendOverlap->view("Send overlap for partition");
824: recvOverlap->view("Receive overlap for partition");
825: }
826: // 2) Partition the mesh
827: return Partitioner::partitionSieve(topology, dim);
828: };
831: // Partition a topology on process 0 and scatter to all processes
832: static void scatterTopology(const Obj<topology_type>& topology, const int dim, const Obj<topology_type>& topologyNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap, const std::string& partitioner, const Obj<topology_type>& subTopology = NULL, const Obj<topology_type>& subTopologyNew = NULL) {
833: if (partitioner == "chaco") {
834: #ifdef PETSC_HAVE_CHACO
835: typedef typename ALECompat::New::Chaco::Partitioner<topology_type> Partitioner;
836: typedef typename ALECompat::New::Partitioner<topology_type> GenPartitioner;
837: typedef typename Partitioner::part_type part_type;
839: part_type *assignment = scatterTopology<Partitioner>(topology, dim, topologyNew, sendOverlap, recvOverlap);
840: if (!subTopology.isNull() && !subTopologyNew.isNull()) {
841: part_type *subAssignment = GenPartitioner::subordinatePartition(topology, 1, subTopology, assignment);
842: Obj<send_overlap_type> subSendOverlap = new send_overlap_type(topology->comm(), topology->debug());
843: Obj<recv_overlap_type> subRecvOverlap = new recv_overlap_type(topology->comm(), topology->debug());
844: const typename topology_type::patch_type patch = 0;
845: const Obj<sieve_type>& sieve = subTopology->getPatch(patch);
846: const Obj<sieve_type>& sieveNew = new Mesh::sieve_type(subTopology->comm(), subTopology->debug());
847: const int numCells = subTopology->heightStratum(patch, 0)->size();
849: createPartitionOverlap(subTopology, subSendOverlap, subRecvOverlap);
850: subTopologyNew->setPatch(0, sieveNew);
851: sieveCompletion::scatterSieve(subTopology, sieve, dim, sieveNew, subSendOverlap, subRecvOverlap, numCells, subAssignment);
852: subTopologyNew->stratify();
853: subTopologyNew->setDistSendOverlap(subSendOverlap);
854: subTopologyNew->setDistRecvOverlap(subRecvOverlap);
855: if (subAssignment != NULL) delete [] subAssignment;
856: }
857: if (assignment != NULL) delete [] assignment;
858: #else
859: throw ALE::Exception("Chaco is not installed. Reconfigure with the flag --download-chaco");
860: #endif
861: } else if (partitioner == "parmetis") {
862: #ifdef PETSC_HAVE_PARMETIS
863: typedef typename ALECompat::New::ParMetis::Partitioner<topology_type> Partitioner;
864: typedef typename Partitioner::part_type part_type;
866: part_type *assignment = scatterTopology<Partitioner>(topology, dim, topologyNew, sendOverlap, recvOverlap);
867: if (assignment != NULL) delete [] assignment;
868: #else
869: throw ALE::Exception("ParMetis is not installed. Reconfigure with the flag --download-parmetis");
870: #endif
871: } else {
872: throw ALE::Exception("Unknown partitioner");
873: }
874: };
875: template<typename Partitioner>
876: static typename Partitioner::part_type *scatterTopology(const Obj<topology_type>& topology, const int dim, const Obj<topology_type>& topologyNew, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
877: typename Partitioner::part_type *assignment = createAssignment<Partitioner>(topology, dim, sendOverlap, recvOverlap);
878: const typename topology_type::patch_type patch = 0;
879: const Obj<sieve_type>& sieve = topology->getPatch(patch);
880: const Obj<sieve_type>& sieveNew = topologyNew->getPatch(patch);
881: const int numCells = topology->heightStratum(patch, 0)->size();
883: sieveCompletion::scatterSieve(topology, sieve, dim, sieveNew, sendOverlap, recvOverlap, numCells, assignment);
884: topologyNew->stratify();
885: return assignment;
886: };
889: static Obj<Mesh> distributeMesh(const Obj<Mesh>& serialMesh, const std::string& partitioner = "chaco") {
890: Obj<Mesh> parallelMesh = new Mesh(serialMesh->comm(), serialMesh->getDimension(), serialMesh->debug());
891: const Obj<Mesh::topology_type>& serialTopology = serialMesh->getTopology();
892: const Obj<Mesh::topology_type>& parallelTopology = new Mesh::topology_type(serialMesh->comm(), serialMesh->debug());
893: const Obj<Mesh::topology_type>& tractionTopology = new Mesh::topology_type(serialMesh->comm(), serialMesh->debug());
894: const Obj<Mesh::sieve_type>& sieve = new Mesh::sieve_type(serialMesh->comm(), serialMesh->debug());
895: const int dim = serialMesh->getDimension();
897: //if (serialMesh->getDistributed()) return serialMesh;
898: ALE_LOG_EVENT_BEGIN;
899: parallelTopology->setPatch(0, sieve);
900: parallelMesh->setTopology(parallelTopology);
901: if (serialMesh->debug()) {
902: serialMesh->view("Serial topology");
903: }
905: // Distribute cones
906: Obj<send_overlap_type> sendOverlap = new send_overlap_type(serialTopology->comm(), serialTopology->debug());
907: Obj<recv_overlap_type> recvOverlap = new recv_overlap_type(serialTopology->comm(), serialTopology->debug());
908: if (serialMesh->hasRealSection("traction")) {
909: const Obj<Mesh::real_section_type>& traction = serialMesh->getRealSection("traction");
911: scatterTopology(serialTopology, dim, parallelTopology, sendOverlap, recvOverlap, partitioner, traction->getTopology(), tractionTopology);
912: } else {
913: scatterTopology(serialTopology, dim, parallelTopology, sendOverlap, recvOverlap, partitioner);
914: }
915: parallelTopology->setDistSendOverlap(sendOverlap);
916: parallelTopology->setDistRecvOverlap(recvOverlap);
918: // Distribute labels
919: const typename topology_type::labels_type& labels = serialTopology->getLabels();
921: for(typename topology_type::labels_type::const_iterator l_iter = labels.begin(); l_iter != labels.end(); ++l_iter) {
922: for(typename topology_type::label_type::const_iterator pl_iter = l_iter->second.begin(); pl_iter != l_iter->second.end(); ++pl_iter) {
923: if (parallelTopology->hasLabel(l_iter->first, pl_iter->first)) continue;
924: const Obj<typename topology_type::patch_label_type>& serialLabel = pl_iter->second;
925: const Obj<typename topology_type::patch_label_type>& parallelLabel = parallelTopology->createLabel(pl_iter->first, l_iter->first);
926: // Create local label
927: const Obj<typename topology_type::patch_label_type::traits::baseSequence>& base = serialLabel->base();
928: const Obj<typename topology_type::sieve_type>& parallelSieve = parallelTopology->getPatch(pl_iter->first);
930: for(typename topology_type::patch_label_type::traits::baseSequence::iterator b_iter = base->begin(); b_iter != base->end(); ++b_iter) {
931: if (parallelSieve->capContains(*b_iter) || parallelSieve->baseContains(*b_iter)) {
932: parallelLabel->addArrow(*serialLabel->cone(*b_iter)->begin(), *b_iter);
933: }
934: }
935: // Get remote labels
936: sieveCompletion::scatterCones(serialLabel, parallelLabel, sendOverlap, recvOverlap);
937: }
938: }
940: // Distribute sections
941: Obj<std::set<std::string> > sections = serialMesh->getRealSections();
943: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
944: if (*name == "traction") {
945: parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), tractionTopology, sendOverlap, recvOverlap));
946: } else {
947: parallelMesh->setRealSection(*name, distributeSection(serialMesh->getRealSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
948: }
949: }
950: sections = serialMesh->getIntSections();
951: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
952: parallelMesh->setIntSection(*name, distributeSection(serialMesh->getIntSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
953: }
954: sections = serialMesh->getPairSections();
955: for(std::set<std::string>::iterator name = sections->begin(); name != sections->end(); ++name) {
956: parallelMesh->setPairSection(*name, distributeSection(serialMesh->getPairSection(*name), parallelMesh->getTopology(), sendOverlap, recvOverlap));
957: }
959: // This is necessary since we create types (like PartitionSection) on a subset of processors
960: if (parallelMesh->debug()) {parallelMesh->view("Parallel Mesh");}
961: parallelMesh->setDistributed(true);
962: ALE_LOG_EVENT_END;
963: return parallelMesh;
964: };
967: template<typename Section>
968: static Obj<Section> distributeSection(const Obj<Section>& serialSection, const Obj<Mesh::topology_type>& parallelTopology, const Obj<send_overlap_type>& sendOverlap, const Obj<recv_overlap_type>& recvOverlap) {
969: if (serialSection->debug()) {
970: serialSection->view("Serial Section");
971: }
972: typedef OverlapValues<send_overlap_type, typename sieveCompletion::topology_type, typename Section::value_type> send_section_type;
973: typedef OverlapValues<recv_overlap_type, typename sieveCompletion::topology_type, typename Section::value_type> recv_section_type;
974: typedef SizeSection<Section> SectionSizer;
975: // TEST THIS! I think this is unnecessary
976: typedef PatchlessSection<Section> SectionFiller;
977: Obj<Section> parallelSection = new Section(parallelTopology);
978: const typename Section::patch_type patch = 0;
979: const Obj<send_section_type> sendSection = new send_section_type(serialSection->comm(), serialSection->debug());
980: const Obj<recv_section_type> recvSection = new recv_section_type(serialSection->comm(), sendSection->getTag(), serialSection->debug());
981: const Obj<SectionSizer> sizer = new SectionSizer(serialSection, patch);
982: const Obj<SectionFiller> filler = new SectionFiller(serialSection, patch);
984: updateSectionLocal(serialSection, parallelSection);
985: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
986: updateSectionRemote(recvOverlap, recvSection, parallelSection);
987: if (parallelSection->debug()) {
988: parallelSection->view("Parallel Section");
989: }
990: return parallelSection;
991: };
994: template<typename Section>
995: static void completeSection(const Obj<Section>& section) {
996: typedef typename Section::topology_type topology_type;
997: typedef typename Distribution<topology_type>::sieveCompletion sieveCompletion;
998: typedef typename topology_type::send_overlap_type send_overlap_type;
999: typedef typename topology_type::recv_overlap_type recv_overlap_type;
1000: typedef typename Section::value_type value_type;
1001: typedef OverlapValues<send_overlap_type, typename sieveCompletion::topology_type, value_type> send_section_type;
1002: typedef OverlapValues<recv_overlap_type, typename sieveCompletion::topology_type, value_type> recv_section_type;
1003: typedef SizeSection<Section> SectionSizer;
1004: typedef PatchlessSection<Section> SectionFiller;
1005: const Obj<topology_type>& topology = section->getTopology();
1006: const typename topology_type::patch_type patch = 0;
1007: const int debug = section->debug();
1008: topology->constructOverlap(patch);
1010: const Obj<send_overlap_type> sendOverlap = topology->getSendOverlap();
1011: const Obj<recv_overlap_type> recvOverlap = topology->getRecvOverlap();
1012: const Obj<send_section_type> sendSection = new send_section_type(section->comm(), section->debug());
1013: const Obj<recv_section_type> recvSection = new recv_section_type(section->comm(), sendSection->getTag(), section->debug());
1014: const Obj<SectionSizer> sizer = new SectionSizer(section, patch);
1015: const Obj<SectionFiller> filler = new SectionFiller(section, patch);
1017: sectionCompletion::completeSection(sendOverlap, recvOverlap, sizer, filler, sendSection, recvSection);
1018: // Update section with remote data
1019: const Obj<typename recv_overlap_type::traits::baseSequence> recvPoints = topology->getRecvOverlap()->base();
1021: for(typename recv_overlap_type::traits::baseSequence::iterator r_iter = recvPoints->begin(); r_iter != recvPoints->end(); ++r_iter) {
1022: const Obj<typename recv_overlap_type::traits::coneSequence>& recvPatches = recvOverlap->cone(*r_iter);
1023: const typename recv_overlap_type::traits::coneSequence::iterator end = recvPatches->end();
1025: for(typename recv_overlap_type::traits::coneSequence::iterator p_iter = recvPatches->begin(); p_iter != end; ++p_iter) {
1026: if (recvSection->getFiberDimension(*p_iter, p_iter.color())) {
1027: if (debug) {std::cout << "["<<section->commRank()<<"]Completed point " << *r_iter << std::endl;}
1028: section->updateAddPoint(patch, *r_iter, recvSection->restrictPoint(*p_iter, p_iter.color()));
1029: }
1030: }
1031: }
1032: };
1033: };
1034: }
1035: }
1036: #endif