cgma
CGMMemFile.cpp
Go to the documentation of this file.
00001 //#include <vector>
00002 #include <set>
00003 #include "string.h"
00004 #include "CubitDefines.h"
00005 #include "CGMMemFile.hpp"
00006 #include "CubitMessage.hpp"
00007 #include "GeometryQueryTool.hpp"
00008 #include "TDUniqueId.hpp"
00009 #include "Body.hpp"
00010 #include "ProcData.hpp"
00011 #include "TDParallel.hpp"
00012 
00013 //std::vector<int> CGMMemFile::bodyToProc;
00014 DLIList <int> CGMMemFile::bodyToProc;
00015 //int *CGMMemFile::bodyToProc = NULL;
00016 //int CGMMemFile::nBodyToProc = 0
00017 
00018 /*
00019 int larger_vol(RefEntity* &x, RefEntity* &y)
00020 {
00021   double x_vol = x->measure();
00022   double y_vol = y->measure();
00023   if (x_vol > y_vol)
00024     return -1;
00025   else if (x_vol < y_vol)
00026     return 1;
00027   else
00028     return 0;
00029 }
00030 */
00031 CGMMemFile::CGMMemFile(void* buffer,
00032                        size_t bufSize)
00033 {
00034   PRINT_DEBUG_100("Initializing memfile in proc %d, size %d.\n", 
00035                   ProcData::instance()->myRank, bufSize);
00036   m_pBuffer = (unsigned char *) buffer;
00037   m_sizeBuffer = bufSize;
00038   m_currentPosition = 0;
00039   //balanceSurf = 0;
00040   //totalLoads = NULL;
00041   //balancedLists = NULL;
00042 }
00043 
00044 CGMMemFile::~CGMMemFile() 
00045 {
00046   // delete the buffer
00047   delete m_pBuffer;
00048 }
00049 
00050 CubitStatus CGMMemFile::bcast_buffer() 
00051 {
00052     //- broadcasts the buffer contained in this object
00053     // get a ProcData object, then simply broadcast
00054   ProcData *proc_data = ProcData::instance();
00055   if (proc_data->is_master()) {
00056     PRINT_DEBUG_100("Broadcasting buffer size from master.\n");
00057 #ifdef USE_MPI
00058     MPI_Bcast(&m_sizeBuffer, 1, MPI_INT, proc_data->masterRank, 
00059               MPI_COMM_WORLD);
00060 #endif
00061     PRINT_DEBUG_100("Broadcasting buffer from master, %d bytes.\n", m_sizeBuffer);
00062 #ifdef USE_MPI
00063     MPI_Bcast(m_pBuffer, m_sizeBuffer, MPI_BYTE, proc_data->masterRank, 
00064               MPI_COMM_WORLD);
00065 #endif
00066   }
00067   else {
00068     int this_size;
00069     PRINT_DEBUG_100("Broadcasting buffer size from proc %d.\n", ProcData::instance()->myRank);
00070 #ifdef USE_MPI
00071     MPI_Bcast(&this_size, 1, MPI_INT, proc_data->masterRank, 
00072               MPI_COMM_WORLD);
00073 #endif
00074     PRINT_DEBUG_100("Processor %d: received size of %d.\n", proc_data->myRank, this_size);
00075     check_size(this_size);
00076     PRINT_DEBUG_100("Broadcasting buffer from proc %d, %d bytes.\n", 
00077                     ProcData::instance()->myRank, this_size);
00078 #ifdef USE_MPI
00079     MPI_Bcast(m_pBuffer, this_size, MPI_BYTE, proc_data->masterRank, 
00080               MPI_COMM_WORLD);
00081 #endif
00082   }
00083   
00084   return CUBIT_SUCCESS;
00085 }
00086 
00087 CubitStatus CGMMemFile::bcast_entity_list(DLIList<RefEntity*> &ref_entity_list) 
00088 {
00089     //- broadcasts the entity list; root takes list as input, 
00090     //- others write as output
00091   CubitStatus status;
00092   double twrite, tbcast;
00093   if (ProcData::instance()->is_master()) {
00094     if (DEBUG_FLAG(137)) {
00095 #ifdef USE_MPI
00096       twrite = MPI_Wtime();
00097 #endif
00098       tbcast = twrite;
00099     }
00100     write_refentity_list(ref_entity_list);
00101     if (CUBIT_TRUE == DEBUG_FLAG(137))
00102 #ifdef USE_MPI
00103       twrite = MPI_Wtime() - twrite;
00104 #endif
00105   }
00106   
00107   status = bcast_buffer();
00108   
00109   if (!ProcData::instance()->is_master()) {
00110     read_refentity_list(ref_entity_list);
00111   }
00112 
00113 /*
00114   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
00115     MPI_Barrier(MPI_COMM_WORLD);
00116     if (ProcData::instance()->is_master()) {
00117       tbcast = MPI_Wtime() - tbcast;
00118       GeometryQueryTool *gti = GeometryQueryTool::instance();
00119       
00120       PRINT_DEBUG_137("buffer size (bytes), time to write buffer, total time to bcast (sec) = %d, %f, %f\n",
00121                       m_sizeBuffer, twrite, tbcast);
00122       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
00123                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00124                       gti->num_bodies());
00125       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
00126                       filename, ProcData::instance()->numProcs, m_sizeBuffer, twrite, tbcast, 
00127                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00128                       gti->num_bodies());
00129     }
00130   }
00131 */
00132   return status;
00133 }
00134 
00135 CubitStatus CGMMemFile::bcast_and_delete_entity_list(DLIList<RefEntity*> &ref_entity_list,
00136                                                      DLIList<RefEntity*> &ref_entity_list_master) 
00137 {
00138   // reset the attribImporteds flags to facilitate attribute reporting
00139   //CubitAttrib::clear_attrib_importeds();
00140 
00141   //- broadcasts the entity list and delete the rest which this process doesn't need
00142   CubitStatus status;
00143   int nEntity, totalRefSize, endEntity, index, nBarEntity, restEntity;
00144   double tbcastDelete;
00145   ProcData *proc_data = ProcData::instance();
00146   int nProcs = proc_data->numProcs;
00147   int myid = proc_data->myRank;
00148 
00149   if (proc_data->is_master()) {
00150     totalRefSize = get_refentity_list_size(ref_entity_list);
00151 
00152     if (DEBUG_FLAG(137)) {
00153 #ifdef USE_MPI
00154       tbcastDelete = MPI_Wtime();
00155 #endif
00156     }
00157     write_refentity_list(ref_entity_list);
00158   }
00159 
00160   status = bcast_buffer();
00161 
00162   if (!proc_data->is_master()) {
00163     read_refentity_list(ref_entity_list);
00164     
00165     nEntity = ref_entity_list.size();
00166     nBarEntity = nEntity/nProcs;
00167     restEntity = nEntity%nProcs;
00168     
00169     // distribute with Round-Robin
00170     // tjt - don't need the 'else' part of the next if test,
00171     // since if restEntity == 0 you'll get the same results (i.e. the 'else' clause
00172     // of the next (nested) if will be the one evaluated
00173     // H.J.K - I thought another 'else' part can reduce some flops.
00174     //if (restEntity) {
00175     if (myid < restEntity) {
00176       index = myid*(nBarEntity+1) + 1;
00177       endEntity = (myid+1)*(nBarEntity+1) + 1;
00178     }
00179     else {
00180       index = restEntity*(nBarEntity+1) + (myid-restEntity)*nBarEntity + 1;
00181       endEntity = restEntity*(nBarEntity+1) + (myid-restEntity+1)*nBarEntity + 1;
00182     }
00183     //}
00184     /* else {
00185        index = myid*nBarEntity + 1;
00186        endEntity = (myid+1)*nBarEntity + 1;
00187        }*/
00188     
00189     // tjt - huh?  Where do you delete the entities?
00190     // H.J.K - I changed code and deleted ref_entity_list directly without temp_list
00191     ref_entity_list.reset();
00192     for (int i = 1; i < index; i++) {
00193       ref_entity_list.remove();
00194     }
00195     
00196     ref_entity_list.step(endEntity-index);
00197     
00198     for (int i = endEntity; i < nEntity+1; i++) {
00199       ref_entity_list.remove();
00200     }
00201 
00202     // report attribs imported
00203     //CubitAttrib::report_attrib_importeds();
00204   }
00205   else {
00206     nEntity = ref_entity_list.size();
00207     nBarEntity = nEntity/nProcs;
00208     restEntity = nEntity%nProcs;
00209 
00210     ref_entity_list_master.clean_out();
00211     ref_entity_list.reset();
00212 
00213     for (int j = 0; j < nBarEntity; j++)
00214       ref_entity_list_master.append(ref_entity_list.get_and_step());
00215 
00216     if (restEntity > 0)
00217       ref_entity_list_master.append(ref_entity_list.get_and_step());
00218 
00219     //cout << "ref_entity_list_master.size()="<< ref_entity_list_master.size() << endl;
00220   }
00221 
00222 /*  
00223   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
00224     MPI_Barrier(MPI_COMM_WORLD);
00225     if (ProcData::instance()->is_master()) {
00226       tbcastDelete = MPI_Wtime() - tbcastDelete;
00227       GeometryQueryTool *gti = GeometryQueryTool::instance();
00228       
00229       PRINT_DEBUG_137("total time to bcast and delete(sec) = %f\n", tbcastDelete);
00230       PRINT_DEBUG_137("total reference entity list size (bytes) = %d\n", totalRefSize);
00231       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
00232                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00233                       gti->num_bodies());
00234       double twrite = 0.0;
00235       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
00236                       filename, ProcData::instance()->numProcs, m_sizeBuffer, twrite, tbcastDelete, 
00237                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(),                       gti->num_bodies());
00238     }
00239   }
00240 */
00241   return status;
00242 }
00243 /*
00244 CubitStatus CGMMemFile::send_body_to_procs(DLIList<RefEntity*> &ref_entity_list) 
00245 {
00246   int nEntity;
00247   DLIList<RefEntity*> temp_list, temp_ref_list;
00248 
00249   ProcData *proc_data = ProcData::instance();
00250 
00251   int nProcs = proc_data->numProcs;
00252 
00253   if (proc_data->is_master()) {
00254     nEntity = ref_entity_list.size();
00255     int nBarEntity = nEntity/nProcs;
00256     int restEntity = nEntity%nProcs;
00257     int nEndEntity = nBarEntity + restEntity;
00258     
00259     ref_entity_list.reset();
00260     bodyToProc.clean_out();
00261 
00262     // make temporary lists to contain geometry information for each processors
00263     for (int i = 0; i < nProcs; i++) {
00264       for ( int j = 0; j < nBarEntity; j++) {
00265     RefEntity* body_ptr = ref_entity_list.get_and_step();
00266     temp_list.append(body_ptr);
00267     bodyToProc.append(TDUniqueId::get_unique_id(body_ptr));
00268     bodyToProc.append(i);
00269       }
00270       
00271       if (i < restEntity) {
00272     RefEntity* body_ptr = ref_entity_list.get_and_step();
00273     temp_list.append(body_ptr);
00274     bodyToProc.append(TDUniqueId::get_unique_id(body_ptr));
00275     bodyToProc.append(i);
00276       }
00277 
00278       temp_list.clean_out();
00279     }
00280   }
00281 
00282   // broadcast number of bodies
00283   MPI_Bcast(&nEntity, 1, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00284 
00285   int body_to_proc[2*nEntity];
00286 
00287   if (proc_data->is_master()) // make temporary array
00288     bodyToProc.copy_to(body_to_proc);
00289   
00290   MPI_Bcast(body_to_proc, 2*nEntity, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00291 
00292   if (!proc_data->is_master()) // copy from temporary array
00293     bodyToProc.copy_from(body_to_proc, 2*nEntity);
00294   
00295   return CUBIT_SUCCESS;
00296 }
00297 */
00298 
00299 CubitStatus CGMMemFile::scatter_entity_list(DLIList<RefEntity*> &ref_entity_list) 
00300 {
00301   // scatter exact amount of geometry information to each processors
00302   CubitStatus status;
00303   double tScatter;
00304   int mySendCount;
00305   int nEntity;
00306   DLIList<RefEntity*> temp_list, temp_ref_list;
00307 
00308   ProcData *proc_data = ProcData::instance();
00309   if (DEBUG_FLAG(137) && proc_data->is_master())
00310 #ifdef USE_MPI
00311     tScatter = MPI_Wtime();
00312 #endif
00313 
00314   int nProcs = proc_data->numProcs;
00315   int *sendCounts = new int[nProcs];
00316   int *displacements = new int[nProcs];
00317   int nBarEntity;
00318   int restEntity;
00319   int nEndEntity;
00320   displacements[0] = 0;
00321 
00322   if (proc_data->is_master()) {
00323     nEntity = ref_entity_list.size();
00324     nBarEntity = nEntity/nProcs;
00325     restEntity = nEntity%nProcs;
00326     nEndEntity = nBarEntity + restEntity;
00327     
00328     ref_entity_list.reset();
00329     int sum = 0;
00330 
00331     // make temporary lists to contain geometry information for each processors
00332     for (int i = 0; i < nProcs; i++) {
00333       for ( int j = 0; j < nBarEntity; j++) {
00334     RefEntity* body_ptr = ref_entity_list.get_and_step();
00335     temp_list.append(body_ptr);
00336       }
00337       
00338       if (i < restEntity) {
00339     RefEntity* body_ptr = ref_entity_list.get_and_step();
00340     temp_list.append(body_ptr);
00341       }
00342 
00343       sendCounts[i] = get_refentity_list_size(temp_list);
00344       sum += sendCounts[i];
00345       temp_list.clean_out();
00346     }
00347 
00348       // check the size of the buffer and resize if necessary
00349     check_size(sum);
00350     
00351       // now actually append the information
00352     ref_entity_list.reset();
00353     for (int i = 0; i < nProcs; i++) {
00354       for ( int j = 0; j < nBarEntity; j++) {
00355     temp_list.append(ref_entity_list.get_and_step());
00356       }
00357       
00358       if (i < restEntity) {
00359     temp_list.append(ref_entity_list.get_and_step());
00360       }
00361 
00362       append_refentity_list(temp_list, sendCounts[i]);
00363       temp_list.clean_out();
00364     }
00365   }
00366 
00367   // broadcast buffer size array
00368   PRINT_DEBUG_100("Broadcasting buffer size array from master.\n");
00369 #ifdef USE_MPI
00370   MPI_Bcast(sendCounts, nProcs, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00371 #endif
00372   
00373   for (int i = 1; i < nProcs; i++) {
00374     displacements[i] = displacements[i-1] + sendCounts[i-1];
00375   }
00376   
00377   mySendCount = sendCounts[proc_data->myRank];
00378 
00379   if (!proc_data->is_master()) check_size(mySendCount);
00380 
00381   PRINT_DEBUG_100("Scattering buffer from master.\n");
00382 
00383   // scatter geometry
00384 #ifdef USE_MPI
00385   MPI_Scatterv(m_pBuffer, sendCounts, displacements, MPI_BYTE, m_pBuffer, 
00386            mySendCount, MPI_BYTE, proc_data->masterRank, MPI_COMM_WORLD);
00387 #endif
00388 
00389   if (!proc_data->is_master()) {
00390     status = read_refentity_list(ref_entity_list);
00391   }
00392   /*
00393   ref_entity_list.reset();
00394   for (int i = 0; i < ref_entity_list.size(); i++) {
00395     DLIList<RefEntity*> entities;
00396     RefEntity *entity = ref_entity_list.get_and_step();
00397     entity->get_all_child_ref_entities(entities);
00398     //cout << "my_id=" << ProcData::instance()->myRank 
00399     // << " num of children=" << entities.size() << endl;
00400     //entities.reset();
00401     //for (int j = 0; j < entities.size(); j++) {
00402     //cout << "my_id=" << ProcData::instance()->myRank 
00403     //   << " measure=" << entities.get_and_step()->measure() << endl;
00404     //}
00405   }
00406   */
00407   /*
00408   // delete unneeded part in master node
00409   if (proc_data->is_master()) {
00410     int index, endEntity;
00411     if (0 < restEntity) {
00412       index = 1;
00413       endEntity = nBarEntity + 2;
00414     }
00415     else {
00416       index = restEntity*(nBarEntity+1) + 1;
00417       endEntity = nBarEntity + 1;
00418     }
00419     ref_entity_list.reset();
00420     for (int i = 1; i < index; i++) {
00421       ref_entity_list.remove();
00422     }
00423     
00424     ref_entity_list.step(endEntity-index);
00425     
00426     for (int j = endEntity; j < nEntity+1; j++) {
00427       ref_entity_list.remove();
00428     }
00429   }
00430   */
00431   /*
00432   int body_to_proc[2*nEntity];
00433 
00434   if (proc_data->is_master()) // make temporary array
00435     bodyToProc.copy_to(body_to_proc);
00436   
00437   MPI_Bcast(body_to_proc, 2*nEntity, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00438 
00439   if (!proc_data->is_master()) // copy from temporary array
00440     bodyToProc.copy_from(body_to_proc, 2*nEntity);
00441 
00442   cout << "myid=" << proc_data->myRank << " bodyToProc.size() = " << bodyToProc.size() << endl;
00443 
00444   DLIList <int> *body_to_proc_ptr = get_body_to_proc();
00445 
00446   cout << "myid=" << proc_data->myRank << " body_to_proc_ptr->size() = " << body_to_proc_ptr->size() << endl;
00447 
00448   if (!proc_data->is_master())
00449     bodyToProc.resize(2*nEntity);
00450 
00451   MPI_Bcast(bodyToProc, nBodyToProc, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00452 
00453   cout << "myid=" << proc_data->myRank << " bodyToProc.size() = " << bodyToProc.size() << endl;
00454   
00455   std::vector<int> *body_to_proc_ptr = get_body_to_proc();
00456   
00457   cout << "myid=" << proc_data->myRank << " body_to_proc_ptr->size() = " << body_to_proc_ptr->size() << endl;
00458   cout << "myid=" << proc_data->myRank << " nEntity pointer = " << &nEntity << endl;
00459   cout << "myid=" << proc_data->myRank << " body_to_proc_ptr = " << body_to_proc_ptr << endl;
00460   cout << "myid=" << proc_data->myRank << " bodyToProcPtr = " << &bodyToProc << endl;
00461   */  
00462   /*
00463   // broadcast body to processors map
00464   MPI_Bcast(bodyToProc, nBodyToProc, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00465   
00466   if (!proc_data->is_master()) {
00467     for (int k = 0; k < nBodyToProc; k++) {
00468       cout << "bodyToProc[" << k << "]=" << bodyToProc[k] << endl;
00469     }
00470   }
00471   */
00472 /*
00473   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
00474     MPI_Barrier(MPI_COMM_WORLD);
00475     if (proc_data->is_master()) {
00476       tScatter = MPI_Wtime() - tScatter;
00477       GeometryQueryTool *gti = GeometryQueryTool::instance();
00478       
00479       PRINT_DEBUG_137("total time to scatter (sec) = %f\n", tScatter);
00480       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
00481               gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00482               gti->num_bodies());
00483       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
00484                       filename, ProcData::instance()->numProcs, m_sizeBuffer, 0.0, tScatter, 
00485                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00486                       gti->num_bodies());
00487     }
00488   }
00489 */
00490   return status;
00491 }
00492 /*
00493 CubitStatus CGMMemFile::scatter_entity_list(DLIList<RefEntity*> &ref_entity_list,
00494                                             const char *filename) 
00495 {
00496   // scatter exact amount of geometry information to each processors
00497   CubitStatus status;
00498   double tScatter;
00499   int mySendCount;
00500   int nEntity;
00501   DLIList<RefEntity*> temp_list, temp_ref_list;
00502 
00503   ProcData *proc_data = ProcData::instance();
00504   if (DEBUG_FLAG(137) && proc_data->is_master())
00505     tScatter = MPI_Wtime();
00506 
00507   int nProcs = proc_data->numProcs;
00508   int *sendCounts = new int[nProcs];
00509   int *displacements = new int[nProcs];
00510   int nBarEntity;
00511   int restEntity;
00512   int nEndEntity;
00513   displacements[0] = 0;
00514 
00515   if (proc_data->is_master()) {
00516     int curPosition = 0;
00517     nEntity = ref_entity_list.size();
00518     nBarEntity = nEntity/nProcs;
00519     restEntity = nEntity%nProcs;
00520     nEndEntity = nBarEntity + restEntity;
00521     
00522     ref_entity_list.reset();
00523     int sum = 0;
00524 
00525     // make temporary lists to contain geometry information for each processors
00526     for (int i = 0; i < nProcs; i++) {
00527       for ( int j = 0; j < nBarEntity; j++) {
00528     RefEntity* body_ptr = ref_entity_list.get_and_step();
00529     temp_list.clean_out();
00530     temp_list.append(body_ptr);
00531     sendCounts[i] += get_refentity_list_size(temp_list);
00532       }
00533       
00534       if (i < restEntity) {
00535     RefEntity* body_ptr = ref_entity_list.get_and_step();
00536     temp_list.clean_out();
00537     temp_list.append(body_ptr);
00538     sendCounts[i] += get_refentity_list_size(temp_list);
00539       }
00540 
00541       sum += sendCounts[i];
00542     }
00543 
00544       // check the size of the buffer and resize if necessary
00545     check_size(sum);
00546     
00547       // now actually append the information
00548     ref_entity_list.reset();
00549     for (int i = 0; i < nProcs; i++) {
00550       for ( int j = 0; j < nBarEntity; j++) {
00551     temp_list.clean_out();
00552     temp_list.append(ref_entity_list.get_and_step());
00553     int count = get_refentity_list_size(temp_list);
00554     append_refentity_list(temp_list, count);
00555       }
00556       
00557       if (i < restEntity) {
00558     temp_list.clean_out();
00559     temp_list.append(ref_entity_list.get_and_step());
00560     int count = get_refentity_list_size(temp_list);
00561     append_refentity_list(temp_list, count);
00562       }
00563 
00564       //append_refentity_list(temp_list, sendCounts[i]);
00565       //temp_list.clean_out();
00566     }
00567   }
00568 
00569   // broadcast buffer size array
00570   PRINT_DEBUG_100("Broadcasting buffer size array from master.\n");
00571   MPI_Bcast(sendCounts, nProcs, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00572   
00573   for (int i = 1; i < nProcs; i++) {
00574     displacements[i] = displacements[i-1] + sendCounts[i-1];
00575   }
00576   
00577   mySendCount = sendCounts[proc_data->myRank];
00578 
00579   if (!proc_data->is_master()) check_size(mySendCount);
00580 
00581   PRINT_DEBUG_100("Scattering buffer from master.\n");
00582 
00583   // scatter geometry
00584   MPI_Scatterv(m_pBuffer, sendCounts, displacements, MPI_BYTE, m_pBuffer, 
00585            mySendCount, MPI_BYTE, proc_data->masterRank, MPI_COMM_WORLD);
00586 
00587   if (!proc_data->is_master()) {
00588     status = read_refentity_list(ref_entity_list);
00589   }
00590 
00591   ref_entity_list.reset();
00592   for (int i = 0; i < ref_entity_list.size(); i++) {
00593     DLIList<RefEntity*> entities;
00594     RefEntity *entity = ref_entity_list.get_and_step();
00595     entity->get_all_child_ref_entities(entities);
00596     cout << "my_id=" << ProcData::instance()->myRank 
00597      << " num of children=" << entities.size() << endl;
00598     entities.reset();
00599     for (int j = 0; j < entities.size(); j++) {
00600       cout << "my_id=" << ProcData::instance()->myRank 
00601        << " measure=" << entities.get_and_step()->measure() << endl;
00602     }
00603   }
00604 
00605   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
00606     MPI_Barrier(MPI_COMM_WORLD);
00607     if (proc_data->is_master()) {
00608       tScatter = MPI_Wtime() - tScatter;
00609       GeometryQueryTool *gti = GeometryQueryTool::instance();
00610       
00611       PRINT_DEBUG_137("total time to scatter (sec) = %f\n", tScatter);
00612       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
00613               gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00614               gti->num_bodies());
00615       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
00616                       filename, ProcData::instance()->numProcs, m_sizeBuffer, 0.0, tScatter, 
00617                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00618                       gti->num_bodies());
00619     }
00620   }
00621 
00622   return status;
00623 }
00624 */
00625 /*
00626 CubitStatus CGMMemFile::vol_balace_send_body_processors(DLIList<RefEntity*> &ref_entity_list) 
00627 {
00628   ProcData *proc_data = ProcData::instance();
00629   bodyToProc.clean_out();
00630   int nEntity = 0;
00631 
00632   if (proc_data->is_master()) {
00633     int nProcs = proc_data->numProcs;
00634     nEntity = ref_entity_list.size();
00635     balancedLists = new DLIList<RefEntity*>*[nProcs];
00636     
00637     for (int k = 0; k < nProcs; k++)
00638       balancedLists[k] = new DLIList<RefEntity*>;
00639     
00640     totalLoads = new double[nProcs];
00641     ref_entity_list.sort(larger_vol);
00642     ref_entity_list.reset();
00643     
00644     // fill first bodies to processors
00645     for (int l = 0; l < nProcs; l++) {
00646       RefEntity *body_ptr = ref_entity_list.get_and_step();
00647       // just calculate volumes for load
00648       totalLoads[l] = body_ptr->measure();
00649       bodyToProc.append(TDUniqueId::get_unique_id(body_ptr));
00650       bodyToProc.append(l);
00651       balancedLists[l]->append(body_ptr);
00652     }
00653     
00654     // fill the rest
00655     for (int i = nProcs; i < nEntity; i++) {
00656       RefEntity *body_ptr = ref_entity_list.get_and_step();
00657       double min_vol = totalLoads[0];
00658       int min_i = 0;
00659       
00660       for (int j = 1; j < nProcs; j++) {
00661     if (min_vol > totalLoads[j]) {
00662       min_vol = totalLoads[j];
00663       min_i = j;
00664     }
00665       }
00666       
00667       totalLoads[min_i] += body_ptr->measure();
00668       bodyToProc.append(TDUniqueId::get_unique_id(body_ptr));
00669       bodyToProc.append(min_i);
00670       balancedLists[min_i]->append(body_ptr);
00671     }
00672   }
00673 
00674   // broadcast number of bodies
00675   MPI_Bcast(&nEntity, 1, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00676   int body_to_proc[2*nEntity];
00677 
00678   if (proc_data->is_master()) // make temporary array
00679     bodyToProc.copy_to(body_to_proc);
00680 
00681   MPI_Bcast(body_to_proc, 2*nEntity, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00682 
00683   if (!proc_data->is_master()) // copy from temporary array
00684     bodyToProc.copy_from(body_to_proc, 2*nEntity);
00685 
00686   return CUBIT_SUCCESS;
00687 }
00688 
00689 CubitStatus CGMMemFile::send_body_to_procs_balanced(DLIList<RefEntity*> &ref_entity_list) 
00690 {
00691   ProcData *proc_data = ProcData::instance();
00692   int nProcs = proc_data->numProcs;
00693   int nEntity;
00694 
00695   // load balance bodies by volume
00696   if (proc_data->is_master()) {
00697     double *total_volumes = new double[nProcs];
00698     nEntity = ref_entity_list.size();
00699     ref_entity_list.sort(larger_vol);
00700     bodyToProc.clean_out();
00701     ref_entity_list.reset();
00702 
00703     // fill first bodies to processors
00704     for (int l = 0; l < nProcs; l++) {
00705       RefEntity *body_ptr = ref_entity_list.get_and_step();
00706       total_volumes[l] = body_ptr->measure();
00707       bodyToProc.append(TDUniqueId::get_unique_id(body_ptr));
00708       bodyToProc.append(l);
00709     }
00710 
00711     // fill the rest
00712     for (int i = nProcs; i < nEntity; i++) {
00713       RefEntity *body_ptr = ref_entity_list.get_and_step();
00714       double min_vol = total_volumes[0];
00715       int min_i = 0;
00716 
00717       for (int j = 1; j < nProcs; j++) {
00718     if (min_vol > total_volumes[j]) {
00719       min_vol = total_volumes[j];
00720       min_i = j;
00721     }
00722       }
00723       
00724       total_volumes[min_i] += body_ptr->measure();
00725       bodyToProc.append(TDUniqueId::get_unique_id(body_ptr));
00726       bodyToProc.append(min_i);
00727     }
00728 
00729     delete total_volumes;
00730   }
00731 
00732   // broadcast number of bodies
00733   MPI_Bcast(&nEntity, 1, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00734   int body_to_proc[2*nEntity];
00735 
00736   if (proc_data->is_master()) // make temporary array
00737     bodyToProc.copy_to(body_to_proc);
00738 
00739   MPI_Bcast(body_to_proc, 2*nEntity, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00740 
00741   if (!proc_data->is_master()) // copy from temporary array
00742     bodyToProc.copy_from(body_to_proc, 2*nEntity);
00743 
00744   return CUBIT_SUCCESS;
00745 }
00746 */
00747 
00748 /*
00749 CubitStatus CGMMemFile::scatter_vol_balanced_entity_list(DLIList<RefEntity*> &ref_entity_list,
00750                              DLIList<RefEntity*> &ref_entity_list_master,
00751                              const char *filename) 
00752 {
00753   // scatter exact amount of geometry information to each processors
00754   CubitStatus status;
00755   double tScatter;
00756   int mySendCount;
00757   int nEntity;
00758 
00759   ProcData *proc_data = ProcData::instance();
00760   if (DEBUG_FLAG(137) && proc_data->is_master())
00761     tScatter = MPI_Wtime();
00762 
00763   int nProcs = proc_data->numProcs;
00764   int *sendCounts = new int[nProcs];
00765   int *displacements = new int[nProcs];
00766   int nBarEntity;
00767   int restEntity;
00768   int nEndEntity;
00769   displacements[0] = 0;
00770   ref_entity_list_master;
00771 
00772   if (proc_data->is_master()) {
00773     int curPosition = 0;
00774     nEntity = ref_entity_list.size();
00775     nBarEntity = nEntity/nProcs;
00776     restEntity = nEntity%nProcs;
00777     nEndEntity = nBarEntity + restEntity;
00778 
00779     double *body_volumes = new double[nEntity];
00780     double *total_volumes = new double[nProcs];
00781     DLIList<RefEntity*> **load_balace_array = new DLIList<RefEntity*>*[nProcs];
00782 
00783     for (int k = 0; k < nProcs; k++)
00784       load_balace_array[k] = new DLIList<RefEntity*>;
00785 
00786     ref_entity_list.reset();
00787     for (int l = 0; l < nProcs; l++) {
00788       RefEntity *body_ptr = ref_entity_list.get_and_step();
00789       load_balace_array[l]->append(body_ptr);
00790       total_volumes[l] = body_ptr->measure();
00791     }
00792 
00793     for (int i = nProcs; i < nEntity; i++) {
00794       RefEntity *body_ptr = ref_entity_list.get_and_step();
00795       double min_vol = total_volumes[0];
00796       int min_i = 0;
00797 
00798       for (int j = 1; j < nProcs; j++) {
00799     if (min_vol > total_volumes[j]) {
00800       min_vol = total_volumes[j];
00801       min_i = j;
00802     }
00803       }
00804       
00805       load_balace_array[min_i]->append(body_ptr);
00806       total_volumes[min_i] += body_ptr->measure();
00807     }
00808 
00809     int sum = 0;
00810     // make temporary lists to contain geometry information for each processors
00811     for (int i = 0; i < nProcs; i++) {
00812       PRINT_DEBUG_100("load_balace_array[i]=%d\n", load_balace_array[i]);
00813       PRINT_DEBUG_100("*load_balace_array[i]=%d\n", *load_balace_array[i]);
00814       PRINT_DEBUG_100("sendCounts[i]=%d\n", sendCounts[i]);
00815       sendCounts[i] = get_refentity_list_size(*(load_balace_array[i]));
00816       sum += sendCounts[i];
00817     }
00818 
00819     // check the size of the buffer and resize if necessary
00820     check_size(sum);
00821     
00822     // now actually append the information
00823     ref_entity_list.clean_out();
00824     for (int i = 0; i < nProcs; i++) {
00825       append_refentity_list(*load_balace_array[i], sendCounts[i]);
00826       ref_entity_list += *load_balace_array[i]; // serialize ref_entity_list
00827     }
00828     
00829     // check and save ref_entity_list information for master
00830     ref_entity_list_master.clean_out();
00831     ref_entity_list_master += *load_balace_array[0];
00832     //ref_entity_list_master = *load_balace_array[0];
00833   }
00834 
00835   // broadcast buffer size array
00836   PRINT_DEBUG_100("Broadcasting buffer size array from master.\n");
00837   MPI_Bcast(sendCounts, nProcs, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00838 
00839   for (int i = 1; i < nProcs; i++) {
00840     displacements[i] = displacements[i-1] + sendCounts[i-1];
00841   }
00842   
00843   mySendCount = sendCounts[proc_data->myRank];
00844 
00845   if (!proc_data->is_master()) check_size(mySendCount);
00846 
00847   PRINT_DEBUG_100("Scattering buffer from master.\n");
00848 
00849   // scatter geometry
00850   MPI_Scatterv(m_pBuffer, sendCounts, displacements, MPI_BYTE, m_pBuffer, 
00851            mySendCount, MPI_BYTE, proc_data->masterRank, MPI_COMM_WORLD);
00852 
00853   if (!proc_data->is_master())
00854     status = read_refentity_list(ref_entity_list);
00855 
00856   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
00857     MPI_Barrier(MPI_COMM_WORLD);
00858     if (proc_data->is_master()) {
00859       tScatter = MPI_Wtime() - tScatter;
00860       GeometryQueryTool *gti = GeometryQueryTool::instance();
00861       
00862       PRINT_DEBUG_137("total time to scatter (sec) = %f\n", tScatter);
00863       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
00864               gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00865               gti->num_bodies());
00866       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
00867                       filename, ProcData::instance()->numProcs, m_sizeBuffer, 0.0, tScatter, 
00868                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00869                       gti->num_bodies());
00870     }
00871   }
00872 
00873   return status;
00874 }
00875 */
00876 /*
00877 // before partition input
00878 CubitStatus CGMMemFile::scatter_balanced_entity_list(DLIList<RefEntity*> &ref_entity_list,
00879                              DLIList<RefEntity*> &ref_entity_list_master,
00880                              const char *filename,
00881                              DLIList<RefEntity*> **balanced_lists) 
00882 {
00883   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list1" << endl;
00884   // scatter exact amount of geometry information to each processors
00885   CubitStatus status;
00886   double tScatter;
00887   int mySendCount;
00888   ProcData *proc_data = ProcData::instance();
00889 
00890   if (DEBUG_FLAG(137) && proc_data->is_master())
00891     tScatter = MPI_Wtime();
00892 
00893   int nProcs = proc_data->numProcs;
00894   int *sendCounts = new int[nProcs];
00895   int *displacements = new int[nProcs];
00896   int nBarEntity;
00897   int restEntity;
00898   int nEndEntity;
00899   displacements[0] = 0;
00900   //ref_entity_list_master;
00901 
00902   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list2" << endl;
00903   if (proc_data->is_master()) {
00904     //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list3" << endl;
00905     int sum = 0;
00906 
00907     // make temporary lists to contain geometry information for each processors
00908     for (int i = 0; i < nProcs; i++) {
00909       //cout << "CGMMemFile::scatter_balanced_entity_list3.11" << endl;
00910       sendCounts[i] = get_refentity_list_size(*(balanced_lists[i]));
00911       //cout << "sendCounts[" << i << "]=" << sendCounts[i] << ",balanced_lists[i]->size()="
00912       //<< balanced_lists[i]->size() << endl;
00913       sum += sendCounts[i];
00914     }
00915 
00916     //cout << "CGMMemFile::scatter_balanced_entity_list3.1" << endl;
00917     // check the size of the buffer and resize if necessary
00918     check_size(sum);
00919     //cout << "CGMMemFile::scatter_balanced_entity_list3.2" << endl;
00920 
00921     // now actually append the information
00922     ref_entity_list.clean_out();
00923     for (int i = 0; i < nProcs; i++) {
00924       append_refentity_list(*balanced_lists[i], sendCounts[i]);
00925       ref_entity_list += *balanced_lists[i]; // serialize ref_entity_list
00926     }
00927     
00928     //cout << "CGMMemFile::scatter_balanced_entity_list3.3" << endl;
00929     // check and save ref_entity_list information for master
00930     ref_entity_list_master.clean_out();
00931     ref_entity_list_master += *balanced_lists[0];
00932     //cout << "CGMMemFile::scatter_balanced_entity_list3.4" << endl;
00933   }
00934 
00935   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4" << endl;
00936   // broadcast buffer size array
00937   PRINT_DEBUG_100("Broadcast
00938 ing buffer size array from master.\n");
00939   
00940   //for (int n = 0; n < nProcs; n++)
00941   //cout << "sendCounts[" << n << "]=" << sendCounts[n] << endl;
00942 
00943   MPI_Bcast(sendCounts, nProcs, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
00944   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.1" << endl;
00945 
00946   for (int i = 1; i < nProcs; i++) {
00947     displacements[i] = displacements[i-1] + sendCounts[i-1];
00948   }
00949   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.2" << endl;
00950   
00951   mySendCount = sendCounts[proc_data->myRank];
00952   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.3" << endl;
00953 
00954   if (!proc_data->is_master()) check_size(mySendCount);
00955   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.4" << endl;
00956 
00957   PRINT_DEBUG_100("Scattering buffer from master.\n");
00958 
00959   //cout << "CGMMemFile::scatter_balanced_entity_list5" << endl;
00960   // scatter geometry
00961   MPI_Scatterv(m_pBuffer, sendCounts, displacements, MPI_BYTE, m_pBuffer, 
00962            mySendCount, MPI_BYTE, proc_data->masterRank, MPI_COMM_WORLD);
00963 
00964   if (!proc_data->is_master())
00965     status = read_refentity_list(ref_entity_list);
00966 
00967   //cout << "CGMMemFile::scatter_balanced_entity_list6" << endl;
00968   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
00969     //MPI_Barrier(MPI_COMM_WORLD);
00970     if (proc_data->is_master()) {
00971       tScatter = MPI_Wtime() - tScatter;
00972       GeometryQueryTool *gti = GeometryQueryTool::instance();
00973       
00974       PRINT_DEBUG_137("total time to scatter (sec) = %f\n", tScatter);
00975       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
00976               gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00977               gti->num_bodies());
00978       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
00979                       filename, ProcData::instance()->numProcs, m_sizeBuffer, 0.0, tScatter, 
00980                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
00981                       gti->num_bodies());
00982     }
00983   }
00984   //cout << "CGMMemFile::scatter_balanced_entity_list7" << endl;
00985 
00986   return status;
00987 }
00988 */
00989 /*
00990 // partition input without balancedLists
00991 CubitStatus CGMMemFile::scatter_balanced_entity_list(DLIList<RefEntity*> &ref_entity_list,
00992                              DLIList<RefEntity*> &ref_entity_list_master,
00993                              const char *filename)
00994 {
00995   //cout << "CGMMemFile::scatter_balanced_entity_list1"
00996   //   << ",balanced_lists=" << balanced_lists << endl;
00997   // scatter exact amount of geometry information to each processors
00998   CubitStatus status;
00999   double tScatter;
01000   int mySendCount;
01001   ProcData *proc_data = ProcData::instance();
01002 
01003   if (DEBUG_FLAG(137) && proc_data->is_master())
01004     tScatter = MPI_Wtime();
01005 
01006   int nProcs = proc_data->numProcs;
01007   int *sendCounts = new int[nProcs];
01008   int *displacements = new int[nProcs];
01009   //int nBarEntity;
01010   //int restEntity;
01011   //int nEndEntity;
01012   displacements[0] = 0;
01013 
01014   cout << "CGMMemFile::scatter_balanced_entity_list2" << endl;
01015 
01016   if (proc_data->is_master()) {
01017     cout << "CGMMemFile::scatter_balanced_entity_list3.1" << endl;
01018 
01019     DLIList<RefEntity*> **balance_lists = new DLIList<RefEntity*>*[nProcs];
01020     
01021     for (int i = 0; i < nProcs; i++)
01022       balance_lists[i] = new DLIList<RefEntity*>;
01023 
01024     cout << "CGMMemFile::scatter_balanced_entity_list3.2" << endl;
01025 
01026     int n_refs = ref_entity_list.size();
01027     ref_entity_list.reset();
01028 
01029     for (; n_refs > 0; n_refs--) {
01030       cout << "CGMMemFile::scatter_balanced_entity_list3.3" << endl;
01031       RefEntity *entity = ref_entity_list.get_and_step();
01032       TDParallel *td_par = (TDParallel *) entity->get_TD(&TDParallel::is_parallel);
01033 
01034       if (td_par == NULL) {
01035     td_par = new TDParallel(entity);
01036       }
01037 
01038       //cout << "td_par->get_proc_id()=" << td_par->get_proc_id() << endl;
01039       //cout << "balance_lists=" << balance_lists << endl;
01040       //cout << "balance_lists[td_par->get_proc_id()]="
01041       //   << balance_lists[td_par->get_proc_id()] << endl;
01042 
01043       int temp_id = td_par->get_proc_id();
01044 
01045       if (temp_id > -1)
01046     balance_lists[temp_id]->append(entity);
01047     }
01048 
01049     cout << "CGMMemFile::scatter_balanced_entity_list3.4" << endl;
01050     int sum = 0;
01051     for (int j = 0; j < nProcs; j++) {
01052       cout << "CGMMemFile::scatter_balanced_entity_list3.41" << endl;
01053 
01054       sendCounts[j] = get_refentity_list_size(*(balance_lists[j]));
01055       cout << "CGMMemFile::scatter_balanced_entity_list3.42" << endl;
01056       sum += sendCounts[j];
01057       cout << "CGMMemFile::scatter_balanced_entity_list3.43" << endl;
01058     }
01059 
01060     cout << "CGMMemFile::scatter_balanced_entity_list3.5" << endl;
01061     // check the size of the buffer and resize if necessary
01062     check_size(sum);
01063 
01064     cout << "CGMMemFile::scatter_balanced_entity_list3.6" << endl;
01065     // now actually append the information
01066     ref_entity_list.clean_out();
01067     for (int k = 0; k < nProcs; k++) {
01068       append_refentity_list(*balance_lists[k], sendCounts[k]);
01069       ref_entity_list += *balance_lists[k]; // serialize ref_entity_list
01070     }
01071     cout << "CGMMemFile::scatter_balanced_entity_list3.7" << endl;    
01072     // check and save ref_entity_list information for master
01073     ref_entity_list_master.clean_out();
01074     ref_entity_list_master += *balance_lists[0];
01075 
01076     for (int l = 0; l < nProcs; l++)
01077       delete balance_lists[l];
01078     cout << "CGMMemFile::scatter_balanced_entity_list3.8" << endl;
01079     delete [] balance_lists;
01080 
01081   }
01082 
01083   cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4" << endl;
01084   // broadcast buffer size array
01085   PRINT_DEBUG_100("Broadcast
01086 ing buffer size array from master.\n");
01087   
01088   //for (int n = 0; n < nProcs; n++)
01089   //cout << "sendCounts[" << n << "]=" << sendCounts[n] << endl;
01090 
01091   MPI_Bcast(sendCounts, nProcs, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
01092   cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.1" << endl;
01093 
01094   for (int i = 1; i < nProcs; i++) {
01095     displacements[i] = displacements[i-1] + sendCounts[i-1];
01096   }
01097   cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.2" << endl;
01098   
01099   mySendCount = sendCounts[proc_data->myRank];
01100   cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.3" << endl;
01101 
01102   if (!proc_data->is_master()) check_size(mySendCount);
01103   cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.4" << endl;
01104 
01105   PRINT_DEBUG_100("Scattering buffer from master.\n");
01106 
01107   cout << "CGMMemFile::scatter_balanced_entity_list5" << endl;
01108   // scatter geometry
01109   MPI_Scatterv(m_pBuffer, sendCounts, displacements, MPI_BYTE, m_pBuffer, 
01110            mySendCount, MPI_BYTE, proc_data->masterRank, MPI_COMM_WORLD);
01111 
01112   cout << "CGMMemFile::scatter_balanced_entity_list5.1" << endl;
01113   if (!proc_data->is_master())
01114     status = read_refentity_list(ref_entity_list);
01115 
01116   cout << "CGMMemFile::scatter_balanced_entity_list6" << endl;
01117   if (CUBIT_TRUE == DEBUG_FLAG(137)) {
01118     //MPI_Barrier(MPI_COMM_WORLD);
01119     if (proc_data->is_master()) {
01120       tScatter = MPI_Wtime() - tScatter;
01121       GeometryQueryTool *gti = GeometryQueryTool::instance();
01122       
01123       PRINT_DEBUG_137("total time to scatter (sec) = %f\n", tScatter);
01124       PRINT_DEBUG_137("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
01125               gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
01126               gti->num_bodies());
01127       PRINT_DEBUG_137("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
01128                       filename, ProcData::instance()->numProcs, m_sizeBuffer, 0.0, tScatter, 
01129                       gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
01130                       gti->num_bodies());
01131     }
01132   }
01133   cout << "CGMMemFile::scatter_balanced_entity_list7" << endl;
01134 
01135   //for (int l = 0; l < nProcs; l++)
01136   //delete balance_lists[l];
01137 
01138 
01139 
01140   return status;
01141 }
01142 */
01143 // partition input
01144 CubitStatus CGMMemFile::scatter_balanced_entity_list(DLIList<RefEntity*> &ref_entity_list,
01145                                                      DLIList<RefEntity*> &ref_entity_list_master,
01146                                                      DLIList<RefEntity*> **balanced_lists) 
01147 {
01148   //cout << "CGMMemFile::scatter_balanced_entity_list1"
01149   //   << ",balanced_lists=" << balanced_lists << endl;
01150   // scatter exact amount of geometry information to each processors
01151   CubitStatus status;
01152   double tScatter;
01153   int mySendCount;
01154   ProcData *proc_data = ProcData::instance();
01155 
01156   if (DEBUG_FLAG(137) && proc_data->is_master())
01157 #ifdef USE_MPI
01158     tScatter = MPI_Wtime();
01159 #endif
01160 
01161   int nProcs = proc_data->numProcs;
01162   int *sendCounts = new int[nProcs];
01163   int *displacements = new int[nProcs];
01164   //int nBarEntity;
01165   //int restEntity;
01166   //int nEndEntity;
01167   displacements[0] = 0;
01168 
01169   //cout << "CGMMemFile::scatter_balanced_entity_list2" << endl;
01170 
01171   if (proc_data->is_master()) {
01172     //cout << "CGMMemFile::scatter_balanced_entity_list3.1" << endl;
01173 
01174     //DLIList<RefEntity*> **balance_lists = new DLIList<RefEntity*>*[nProcs];
01175     
01176     //for (int i = 0; i < nProcs; i++)
01177     //balance_lists[i] = new DLIList<RefEntity*>;
01178 
01179     //cout << "CGMMemFile::scatter_balanced_entity_list3.4" << endl;
01180     int sum = 0;
01181     for (int j = 0; j < nProcs; j++) {
01182       //cout << "CGMMemFile::scatter_balanced_entity_list3.41" << endl;
01183       //cout << "balanced_lists[j]->size()=" << balanced_lists[j]->size()
01184       //   << endl;
01185 
01186       sendCounts[j] = get_refentity_list_size(*(balanced_lists[j]));
01187       //cout << "CGMMemFile::scatter_balanced_entity_list3.42" << endl;
01188       sum += sendCounts[j];
01189       //cout << "CGMMemFile::scatter_balanced_entity_list3.43" << endl;
01190     }
01191 
01192     //cout << "CGMMemFile::scatter_balanced_entity_list3.5" << endl;
01193     // check the size of the buffer and resize if necessary
01194     check_size(sum);
01195 
01196     //cout << "CGMMemFile::scatter_balanced_entity_list3.6" << endl;
01197     // now actually append the information
01198     ref_entity_list.clean_out();
01199     for (int k = 0; k < nProcs; k++) {
01200       append_refentity_list(*balanced_lists[k], sendCounts[k]);
01201       ref_entity_list += *balanced_lists[k]; // serialize ref_entity_list
01202     }
01203     //cout << "CGMMemFile::scatter_balanced_entity_list3.7" << endl;    
01204     // check and save ref_entity_list information for master
01205     ref_entity_list_master.clean_out();
01206     //cout << "CGMMemFile::scatter_balanced_entity_list3.71" << endl;
01207     //cout << "balanced_lists[0]=" << balanced_lists[0] << endl;
01208     ref_entity_list_master += *balanced_lists[0];
01209     //cout << "CGMMemFile::scatter_balanced_entity_list3.72" << endl;
01210   }
01211 
01212   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4" << endl;
01213   // broadcast buffer size array
01214   PRINT_DEBUG_100("Broadcasting buffer size array from master.\n");
01215   
01216   //for (int n = 0; n < nProcs; n++)
01217   //cout << "sendCounts[" << n << "]=" << sendCounts[n] << endl;
01218 
01219 #ifdef USE_MPI
01220   MPI_Bcast(sendCounts, nProcs, MPI_INT, proc_data->masterRank, MPI_COMM_WORLD);
01221 #endif
01222   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.1" << endl;
01223 
01224   for (int i = 1; i < nProcs; i++) {
01225     displacements[i] = displacements[i-1] + sendCounts[i-1];
01226   }
01227   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.2" << endl;
01228   
01229   mySendCount = sendCounts[proc_data->myRank];
01230   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.3" << endl;
01231 
01232   if (!proc_data->is_master()) check_size(mySendCount);
01233   //cout << "my_id=" << ProcData::instance()->myRank << "CGMMemFile::scatter_balanced_entity_list4.4" << endl;
01234 
01235   PRINT_DEBUG_100("Scattering buffer from master.\n");
01236 
01237   //cout << "CGMMemFile::scatter_balanced_entity_list5" << endl;
01238   // scatter geometry
01239 #ifdef USE_MPI
01240   MPI_Scatterv(m_pBuffer, sendCounts, displacements, MPI_BYTE, m_pBuffer, 
01241            mySendCount, MPI_BYTE, proc_data->masterRank, MPI_COMM_WORLD);
01242 #endif
01243 
01244   //cout << "CGMMemFile::scatter_balanced_entity_list5.1" << endl;
01245   if (!proc_data->is_master())
01246     status = read_refentity_list(ref_entity_list);
01247 
01248   //cout << "CGMMemFile::scatter_balanced_entity_list6" << endl;
01249 //  if (CUBIT_TRUE == DEBUG_FLAG(138)) {
01250     //MPI_Barrier(MPI_COMM_WORLD);
01251 //    if (proc_data->is_master()) {
01252 //      tScatter = MPI_Wtime() - tScatter;
01253 //      GeometryQueryTool *gti = GeometryQueryTool::instance();
01254       
01255 //      PRINT_DEBUG_138("total time to scatter (sec) = %f\n", tScatter);
01256 //      PRINT_DEBUG_138("Complexity: vertices, edges, faces, bodies = %d, %d, %d, %d\n",
01257 //            gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
01258 //            gti->num_bodies());
01259 //      PRINT_DEBUG_138("Stats: %s, %d, %d, %f, %f, %d, %d, %d, %d\n",
01260 //                      filename, ProcData::instance()->numProcs, m_sizeBuffer, 0.0, tScatter, 
01261 //                      gti->num_ref_vertices(), gti->num_ref_edges(), gti->num_ref_faces(), 
01262 //                      gti->num_bodies());
01263 //    }
01264 //  }
01265   //cout << "CGMMemFile::scatter_balanced_entity_list7" << endl;
01266 
01267   //for (int l = 0; l < nProcs; l++)
01268   //delete balance_lists[l];
01269 
01270 
01271 
01272   return status;
01273 }
01274 
01275 CubitStatus CGMMemFile::check_size(int &target_size, CubitBoolean keep) 
01276 {
01277   PRINT_DEBUG_100("Checking buffer size on proc %d, target size %d.\n", 
01278                   ProcData::instance()->myRank, target_size);
01279   if ((int)m_sizeBuffer < target_size) {
01280     PRINT_DEBUG_100("Increasing buffer size on proc %d.\n", ProcData::instance()->myRank);
01281     void *temp_buffer = malloc(target_size);
01282     if (keep && 0 != m_currentPosition) memcpy(temp_buffer, m_pBuffer, m_currentPosition);
01283     delete m_pBuffer;
01284     m_pBuffer = (unsigned char *) temp_buffer;
01285     m_sizeBuffer = target_size;
01286   }
01287 
01288   return CUBIT_SUCCESS;
01289 }
01290 DLIList <int> *CGMMemFile::get_body_to_proc() {
01291   return &bodyToProc;
01292 }
01293 /*
01294 std::vector<int> *CGMMemFile::get_body_to_proc() {
01295   return &bodyToProc;
01296 }
01297 */
01298 /*
01299 int *CGMMemFile::get_body_to_proc() {
01300   return bodyToProc;
01301 }
01302 
01303 int CGMMemFile::get_num_body_to_proc() {
01304   return nBodyToProc;
01305 }
01306 */
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines