Sierra Toolkit  Version of the Day
ZoltanPartition.cpp
1 
2 #include <vector>
3 #include <string>
4 #include <sstream>
5 #include <cstdlib>
6 
8 #include <stk_mesh/base/Field.hpp>
9 #include <stk_mesh/base/FieldData.hpp>
10 #include <stk_mesh/base/Entity.hpp>
11 #include <stk_mesh/base/Bucket.hpp>
12 #include <stk_mesh/base/BulkData.hpp>
13 #include <stk_mesh/fem/FEMMetaData.hpp>
14 
15 #include <stk_util/parallel/ParallelReduce.hpp>
16 
17 #include <zoltan.h>
18 
19 #include <Teuchos_ParameterList.hpp>
20 
21 using namespace std;
22 using namespace stk_classic;
23 using namespace stk_classic::rebalance;
24 
25 #define STK_GEOMDECOMP_DEBUG 0
26 
27 namespace {
28 
29 
30 double static_zoltan_version(const double v=0) {
31  static double version=0;
32  if (v) { version=v;}
33  return version;
34 }
35 
36 inline unsigned num_gid_entries() {
37  static const unsigned n=2;
38  return n;
39 }
40 
41 inline unsigned num_lid_entries() {
42  static const unsigned n=2;
43  return n;
44 }
45 inline unsigned wdim() {
46  static const unsigned n=1;
47  return n;
48 }
49 
50 inline void convert_param_to_string(const Parameters &from,
51  vector < pair<std::string, std::string> > &to)
52 {
53  Parameters::ConstIterator
54  from_iter = from.begin(),
55  from_end = from.end();
56 
57  for (; from_iter != from_end; ++from_iter) {
58  std::string s;
59  const std::string name = from.name(from_iter);
60  const std::string entry = from.entry(from_iter).getValue(&s);
61  std::pair<std::string,std::string> p(name,entry);
62 
63  to.push_back(p);
64  }
65 }
66 
67 inline void fill_parameters (const char *T[][2],
68  const int i,
69  Parameters &Entry)
70 {
71  for (int j=0; j<i; ++j) Entry.set(T[j][0], T[j][1]);
72 }
73 
74 void fill_name_conversion( Parameters & name_conversion )
75 {
76 
77  Parameters & general = name_conversion.sublist("General");
78 
79  general.set("LOAD BALANCING METHOD" , "LB_METHOD");
80  general.set("ZOLTAN DEBUG LEVEL" , "DEBUG_LEVEL");
81  general.set("DEBUG PROCESSOR NUMBER" , "DEBUG_PROCESSOR");
82  general.set("TIMER" , "TIMER");
83  general.set("DETERMINISTIC DECOMPOSITION", "DETERMINISTIC");
84  general.set("DEBUG MEMORY" , "DEBUG_MEMORY");
85  general.set("IMBALANCE TOLERANCE" , "IMBALANCE_TOL");
86  general.set("RENUMBER PARTITIONS" , "REMAP");
87  general.set("KEEP CUTS" , "KEEP_CUTS");
88  general.set("REUSE CUTS" , "RCB_REUSE");
89  general.set("RCB RECOMPUTE BOX" , "RCB_RECOMPUTE_BOX");
90  general.set("CHECK GEOMETRY" , "CHECK_GEOM");
91  general.set("LOCK RCB DIRECTIONS" , "RCB_LOCK_DIRECTIONS");
92  general.set("SET RCB DIRECTIONS" , "RCB_SET_DIRECTIONS");
93  general.set("RCB MAX ASPECT RATIO" , "RCB_MAX_ASPECT_RATIO");
94  general.set("RECTILINEAR RCB BLOCKS" , "RCB_RECTILINEAR_BLOCKS");
95  general.set("OCTREE DIMENSION" , "OCT_DIM");
96  general.set("OCTREE METHOD" , "OCT_METHOD");
97  general.set("OCTREE MIN ENTITIES" , "OCT_MINOBJECTS");
98  general.set("OCTREE MAX ENTITIES" , "OCT_MAXOBJECTS");
99 // // These values are never changed, but must
100  // be set so default values work correctly.
101  general.set("NUMBER GLOBAL ID ENTRIES" , "NUM_GID_ENTRIES");
102  general.set("NUMBER LOCAL ID ENTRIES" , "NUM_LID_ENTRIES");
103  general.set("ENTITY WEIGHT DIMENSION" , "OBJ_WEIGHT_DIM");
104  general.set("RETURN LISTS" , "RETURN_LISTS");
105  general.set("AUTOMATIC MIGRATION" , "AUTO_MIGRATE");
106  general.set("DISTANCE" , "DISTANCE");
107 
108  Parameters & rcb = name_conversion.sublist("0");
109  rcb.set("OVER ALLOCATE MEMORY" , "RCB_OVERALLOC");
110  rcb.set("ALGORITHM DEBUG LEVEL" , "RCB_OUTPUT_LEVEL");
111 
112  Parameters & rib = name_conversion.sublist("1");
113  rib.set("OVER ALLOCATE MEMORY" , "RIB_OVERALLOC");
114  rib.set("ALGORITHM DEBUG LEVEL" , "RIB_OUTPUT_LEVEL");
115 
116  Parameters & hsfc = name_conversion.sublist("2");
117  hsfc.set("OVER ALLOCATE MEMORY" , "");
118  hsfc.set("ALGORITHM DEBUG LEVEL" , "" );
119 
120  Parameters & oct = name_conversion.sublist("3");
121  oct.set("OVER ALLOCATE MEMORY" , "");
122  oct.set("ALGORITHM DEBUG LEVEL" , "OCT_OUTPUT_LEVEL");
123 
124  Parameters & graph = name_conversion.sublist("4");
125  graph.set("OVER ALLOCATE MEMORY" , "");
126  graph.set("ALGORITHM DEBUG LEVEL" , "" );
127 }
128 
129 
130 void fill_value_conversion( Parameters & value_conversion )
131 {
132  Parameters & lb_method = value_conversion.sublist("LOAD BALANCING METHOD");
133  lb_method.set("0" , "RCB");
134  lb_method.set("1" , "RIB");
135  lb_method.set("2" , "HSFC");
136  lb_method.set("3" , "OCTPART");
137  lb_method.set("4" , "GRAPH");
138 
139  Parameters & timer = value_conversion.sublist("TIMER");
140  timer.set("0" , "WALL");
141  timer.set("1" , "CPU");
142 
143 }
144 
145 void fill_default_values( Parameters & values )
146 {
147  Parameters & default_values = values; //values.sublist("General");
148 
149  default_values.set("LOAD BALANCING METHOD" , "0");
150  default_values.set("RENUMBER PARTITIONS" , "1");
151  default_values.set("ZOLTAN DEBUG LEVEL" , "0");
152  default_values.set("TIMER" , "0");
153  default_values.set("DETERMINISTIC DECOMPOSITION", "1");
154  default_values.set("DEBUG MEMORY" , "1");
155  default_values.set("IMBALANCE TOLERANCE" , "1.1");
156  default_values.set("KEEP CUTS" , "1");
157  default_values.set("REUSE CUTS" , "1");
158  default_values.set("OVER ALLOCATE MEMORY" , "1.1");
159  default_values.set("ALGORITHM DEBUG LEVEL" , "0");
160 // default_values.set("OCTREE MIN ENTITIES" , "1");
161 // default_values.set("OCTREE MAX ENTITIES" , "1");
162  default_values.set("NUMBER GLOBAL ID ENTRIES" , "2");
163  default_values.set("NUMBER LOCAL ID ENTRIES" , "2");
164  default_values.set("ENTITY WEIGHT DIMENSION" , "1");
165  default_values.set("RETURN LISTS" , "EXPORT");
166 }
167 
168 
169 
170 #if STK_GEOMDECOMP_DEBUG>=2
171 void debug_print_decomp_export(Zoltan *zoltan,
172  Zoltan *zoltan_id )
173 {
174  int i;
175  int num_export = zoltan->Num_Exported();
176  ZOLTAN_ID_PTR export_lids = zoltan->Export_Local_IDs();
177  ZOLTAN_ID_PTR export_gids = zoltan->Export_Global_IDs();
178  int* export_procs = zoltan->Export_Proc_IDs();
179  int Z_LID_SIZE = zoltan->Num_Lid_Entries();
180  int Z_GID_SIZE = zoltan->Num_Gid_Entries();
181 
182  if ( export_gids!=NULL && export_lids!=NULL && export_procs!=NULL ) {
183  Env::output() << ": Zoltan RCB EXPORTS" << std::endl;
184  for ( i = 0; i < num_export; i++ ) {
185  Env::output()
186  << " " << i
187  << ": GID = "
188  << "T" << zoltan_id->Type( &export_gids[i*Z_GID_SIZE]) << " "
189  << "I" << zoltan_id->Index(&export_gids[i*Z_GID_SIZE]) << " "
190  << "P" << zoltan_id->Proc( &export_gids[i*Z_GID_SIZE]) << " "
191  << " LID = "
192  << "T" << zoltan_id->Type( &export_lids[i*Z_LID_SIZE]) << " "
193  << "I" << zoltan_id->Index(&export_lids[i*Z_LID_SIZE]) << " "
194  << " EXPORT_PROC_ID = "
195  << export_procs[i]
196  << std::endl;
197  }
199  }
200 }
201 #endif
202 
203 
204 
205 extern "C" {
206  int stkCallback_Num_Elements( void *data, int *ierr );
207  void stkCallback_Element_List( void *data,
208  int Num_gid_entries,
209  int Num_lid_entries,
210  ZOLTAN_ID_PTR global_ids,
211  ZOLTAN_ID_PTR local_ids, //{Iterator or index}
212  int wdim,
213  float *weights,
214  int *ierr );
215  int stkCallback_Num_Dimensions( void *data, int *ierr );
216  void stkCallback_Centroid_Coord( void *data,
217  int Num_gid_entries,
218  int Num_lid_entries,
219  ZOLTAN_ID_PTR global_id,
220  ZOLTAN_ID_PTR local_id, //{Iterator or index}
221  double *geom,
222  int *ierr );
223  int stkCallback_Num_Edges( void *data,
224  int Num_gid_entries,
225  int Num_lid_entries,
226  ZOLTAN_ID_PTR global_id,
227  ZOLTAN_ID_PTR local_id, //{Iterator or index}
228  int *ierr );
229  void stkCallback_Edge_List( void *data,
230  int Num_gid_entries,
231  int Num_lid_entries,
232  ZOLTAN_ID_PTR global_id,
233  ZOLTAN_ID_PTR local_id, //{Iterator or index}
234  ZOLTAN_ID_PTR nbor_global_id,
235  int *nbor_procs,
236  int wgt_dim,
237  float *ewgts,
238  int *ierr );
239 }
240 
241 
242 int stkCallback_Num_Elements( void *data, int *ierr )
243 {
244  if ( data == NULL ) {
245  *ierr = ZOLTAN_FATAL; // Set FATAL Zoltan error flag
246  return 0;
247  }
249  stk_classic::rebalance::Zoltan *zdata = dynamic_cast<stk_classic::rebalance::Zoltan*>(gdata);
250 
251  if (zdata == NULL ) {
252  *ierr = ZOLTAN_FATAL; // Set FATAL Zoltan error flag
253  return 0;
254  }
255 
256  int ne = zdata->num_elems();
257 
258  *ierr = ZOLTAN_OK;
259  return ne;
260 }
261 
262 void stkCallback_Element_List( void *data,
263  int Num_gid_entries,
264  int Num_lid_entries,
265  ZOLTAN_ID_PTR global_ids,
266  ZOLTAN_ID_PTR local_ids, //{Iterator or index}
267  int weightdim,
268  float *weights,
269  int *ierr )
270 {
271  if (!data) {
272  *ierr = ZOLTAN_FATAL; // Set FATAL Zoltan error flag
273  return;
274  }
275 
277  stk_classic::rebalance::Zoltan *zdata = dynamic_cast<stk_classic::rebalance::Zoltan*> (gdata);
278 
279  if (!zdata) {
280  *ierr = ZOLTAN_FATAL; // Set FATAL Zoltan error flag
281  return;
282  }
283 
284  unsigned k=0, l=0;
285  const unsigned num_local_ids = zdata->num_moid();
286  for (unsigned j=0; j<num_local_ids; ++j) {
287  local_ids [k] = j;
288  global_ids[k] = 0; // region_id
289  ++k;
290  local_ids [k] = 0;
291  global_ids[k] = static_cast<ZOLTAN_ID_TYPE>(zdata->globalID(j));
292  ++k;
293  if (weightdim) {
294  weights[l++] = zdata->entity_weight(j);
295  } else {
296  ++l;
297  }
298  }
299  *ierr = ZOLTAN_OK;
300  return;
301 
302 }
303 
304 int stkCallback_Num_Dimensions( void *data, int *ierr )
305 {
306  if ( !data ) {
307  *ierr = ZOLTAN_FATAL;
308  return 0;
309  }
311  stk_classic::rebalance::Zoltan *zdata = dynamic_cast<stk_classic::rebalance::Zoltan*> (gdata);
312 
313  if ( !zdata ) {
314  *ierr = ZOLTAN_FATAL;
315  return 0;
316  }
317 
318  const int nd = zdata->spatial_dimension();
319 
320  *ierr = ZOLTAN_OK;
321  return nd;
322 
323 }
324 
325 void stkCallback_Centroid_Coord( void *data,
326  int Num_gid_entries,
327  int Num_lid_entries,
328  ZOLTAN_ID_PTR global_id,
329  ZOLTAN_ID_PTR local_id, //{Iterator or index}
330  double *geom,
331  int *ierr )
332 {
333  std::vector<double> temp(3,0.0);
334 
335  // (from include Fmwk_Sierra_Zoltan_Defines.h:)
336  // ( Num_gid_entries = ZOLTAN_GID_SIZE 2 )
337  // ( Num_lid_entries = ZOLTAN_LID_SIZE 2 )
338 
339  if ( !data ) {
340  *ierr = ZOLTAN_FATAL;
341  return ;
342  }
344  stk_classic::rebalance::Zoltan *zdata = dynamic_cast<stk_classic::rebalance::Zoltan*> (gdata);
345 
346  if ( !zdata ) {
347  *ierr = ZOLTAN_FATAL;
348  return ;
349  }
350 
351  int lid = local_id[ 0 ]; // Local Element ID
352 
353  const mesh::Entity & target_entity = * zdata->mesh_entity( lid );
354  const GeomDecomp::VectorField & coor = * zdata->entity_coord_ref();
355  const unsigned nd = zdata->spatial_dimension();
356 
357  /*
358  * Obtain the centroid coordinates of the element by averaging all
359  * the nodal coordinates of the nodes associated with the element.
360  * Use GeomDecomp friend function, ent( , , )
361  *
362  */
363 
364  rebalance::GeomDecomp::entity_to_point( target_entity, coor, temp );
365 
366  for (size_t i=0 ; i < nd ; i++ ) geom[ i ] = (double) temp[ i ];
367 
368  *ierr = ZOLTAN_OK;
369 }
370 
371 
372 
373 void getNeighbors( const mesh::Entity & entity,
374  std::set<const mesh::Entity*> & nodes )
375 {
377  const stk_classic::mesh::EntityRank element_rank = fem_meta.element_rank();
378 
379  nodes.clear();
380 
381  mesh::PairIterRelation iElem = entity.relations(element_rank);
382 
383  for ( ; iElem.first != iElem.second; ++iElem.first ) {
384  mesh::Entity * elem = iElem.first->entity();
385  mesh::PairIterRelation iNode = elem->relations(fem_meta.node_rank());
386  for ( ; iNode.first != iNode.second; ++iNode.first ) {
387  mesh::Entity * node = iNode.first->entity();
388  if (&entity != node) nodes.insert( node );
389  }
390  }
391 }
392 
393 int numEdges( const mesh::Entity & entity ) {
394 
395  std::set<const mesh::Entity*> nodes;
396 
397  getNeighbors( entity, nodes );
398  return nodes.size();
399 }
400 
401 int stkCallback_Num_Edges( void *data,
402  int Num_gid_entries,
403  int Num_lid_entries,
404  ZOLTAN_ID_PTR global_id,
405  ZOLTAN_ID_PTR local_id, //{Iterator or index}
406  int *ierr )
407 {
408  // (from include Fmwk_Sierra_Zoltan_Defines.h:)
409  // ( Num_gid_entries = ZOLTAN_GID_SIZE 2 )
410  // ( Num_lid_entries = ZOLTAN_LID_SIZE 2 )
411 
412  *ierr = ZOLTAN_OK;
413 
414  if ( !data ) {
415  *ierr = ZOLTAN_FATAL;
416  return 0;
417  }
419  stk_classic::rebalance::Zoltan *zdata = dynamic_cast<stk_classic::rebalance::Zoltan*> (gdata);
420 
421  if ( !zdata ) {
422  *ierr = ZOLTAN_FATAL;
423  return 0;
424  }
425 
426  int lid = local_id[ 0 ]; // Local Element ID
427 
428  const mesh::Entity & target_entity = * zdata->mesh_entity( lid );
429 
430  return numEdges( target_entity );
431 
432 }
433 
434 void stkCallback_Edge_List( void *data,
435  int Num_gid_entries,
436  int Num_lid_entries,
437  ZOLTAN_ID_PTR global_id,
438  ZOLTAN_ID_PTR local_id, //{Iterator or index}
439 
440  // Returned
441  ZOLTAN_ID_PTR nbor_global_id, //owning processor
442  int *nbor_procs,
443  int wgt_dim,
444  float *ewgts,
445 
446  int *ierr )
447 {
448  // (from include Fmwk_Sierra_Zoltan_Defines.h:)
449  // ( Num_gid_entries = ZOLTAN_GID_SIZE 2 )
450  // ( Num_lid_entries = ZOLTAN_LID_SIZE 2 )
451 
452  *ierr = ZOLTAN_OK;
453 
454  if ( !data ) {
455  *ierr = ZOLTAN_FATAL;
456  return ;
457  }
459  stk_classic::rebalance::Zoltan *zdata = dynamic_cast<stk_classic::rebalance::Zoltan*> (gdata);
460 
461  if ( !zdata ) {
462  *ierr = ZOLTAN_FATAL;
463  return ;
464  }
465 
466  int lid = local_id[ 0 ]; // Local Node ID
467 
468  const mesh::Entity & target_entity = * zdata->mesh_entity( lid );
469 
470  std::set<const mesh::Entity*> nodes;
471  getNeighbors( target_entity, nodes );
472 
473  int counter(0);
474  for ( std::set<const mesh::Entity*>::iterator i = nodes.begin();
475  i != nodes.end(); ++i ) {
476  nbor_global_id[counter*2+0] = 0; // region_id
477 
478  const mesh::Entity & n = **i;
479  nbor_global_id[counter*2+1] = n.key().id();
480 
481  nbor_procs[counter] = n.owner_rank();
482 
483  if ( wgt_dim ) {
484  ewgts[counter] = 1;
485  }
486  ++counter;
487  }
488 }
489 
490 }
491 
492 
493 
494 const std::string Zoltan::m_zoltanparametersname_="Zoltan_Parameters";
495 const std::string Zoltan::m_defaultparametersname_="DEFAULT";
496 
497 
498 const std::string Zoltan::zoltan_parameters_name()
499 {
500  return m_zoltanparametersname_;
501 }
502 
503 const std::string Zoltan::default_parameters_name()
504 {
505  return m_defaultparametersname_;
506 }
507 
508 void Zoltan::init_default_parameters()
509 {
510  fill_default_values(m_default_parameters_);
511 }
512 
513 
514 static Parameters *Name_Conversion =NULL;
515 static Parameters *Value_Conversion=NULL;
516 
517 double Zoltan::zoltan_version() const { return static_zoltan_version(); }
518 
519 //: ===========
520 //: Constructor
521 //: ===========
522 
523 namespace {
524 void merge_parameters(std::vector <std::pair<std::string, std::string> > &str_zoltan_params,
525  const Parameters &Zoltan_Params) {
526  Parameters Merged_Zoltan_Params ;
527  Parameters Converted_Zoltan_Params;
528 
529  rebalance::Zoltan::merge_default_values (Zoltan_Params,
530  Merged_Zoltan_Params);
531 
532  rebalance::Zoltan::convert_names_and_values(Merged_Zoltan_Params,
533  Converted_Zoltan_Params);
534 
535  convert_param_to_string (Converted_Zoltan_Params,
536  str_zoltan_params);
537  return;
538 }
539 }
540 
541 Zoltan::Zoltan(ParallelMachine pm, const unsigned ndim, Parameters & rebal_region_parameters, const std::string parameters_name) :
542  GeomDecomp(pm),
543  m_zoltan_id_(NULL),
544  m_spatial_dimension_(ndim),
545  m_total_number_entities_(0)
546 {
547  /* Determine if the default set of parameters already exists. */
548  if( !rebal_region_parameters.isSublist(default_parameters_name()) )
549  {
550  init_default_parameters();
551  rebal_region_parameters.sublist(default_parameters_name()) = m_default_parameters_;
552  }
553 
554  /* If name is empty, use default values */
555  std::string default_name =
556  (parameters_name.empty()) ? default_parameters_name() : parameters_name ;
557 
558  if( !rebal_region_parameters.isSublist(default_name) ) {
559  throw std::runtime_error("The Zoltan parameter set '" + default_name + "' does not exist.");
560  }
561  const Parameters & zoltan_params = rebal_region_parameters.sublist(default_name);
562 
563  std::vector <std::pair<std::string, std::string> > str_zoltan_params;
564  merge_parameters(str_zoltan_params, zoltan_params);
565 
566  init(str_zoltan_params);
567 }
568 
569 void
570 Zoltan::set_mesh_info( const std::vector<mesh::Entity *> &mesh_entities,
571  const VectorField * nodal_coord_ref,
572  const ScalarField * elem_weight_ref)
573 {
574  MeshInfo mesh_info;
575 
576  /* Keep track of the total number of elements. */
577  m_total_number_entities_ = mesh_entities.size();
578 
579  mesh_info.mesh_entities = mesh_entities;
580  mesh_info.nodal_coord_ref = nodal_coord_ref;
581  mesh_info.elem_weight_ref = elem_weight_ref;
582 
588  mesh_info.dest_proc_ids.assign(mesh_entities.size(), stk_classic::parallel_machine_rank(comm_));
589 
590  m_mesh_information_ = mesh_info;
591 }
592 
593 void Zoltan::init( const vector< pair<std::string,std::string> >
594  &dynamicLoadRebalancingParameters ) {
595  if (0==static_zoltan_version()) {
596  const double v = init_zoltan_library();
597  static_zoltan_version(v);
598  }
599 
604  m_zoltan_id_ = Zoltan_Create( comm_ );
605  if ( m_zoltan_id_ == NULL ) {
606  throw runtime_error ("(FATAL ERROR) Zoltan_Create() returned NULL");
607  }
608 
613  vector<pair<std::string,std::string> >::const_iterator
614  P = dynamicLoadRebalancingParameters.begin(),
615  PE = dynamicLoadRebalancingParameters.end();
616 
617  for ( ; PE != P ; P++ ) {
618 
619  char * label = const_cast<char*>( P->first.c_str() ) ;
620  char * value = const_cast<char*>( P->second.c_str() ) ;
621 
622  if (ZOLTAN_OK != (Zoltan_Set_Param(m_zoltan_id_,label,value)))
623  {
624  throw runtime_error(": FATAL ERROR returned from Zoltan_Set_Param ");
625  }
626  }
627 
631  if ( ZOLTAN_OK != register_callbacks() )
632  throw runtime_error ("zoltan->Register_Callbacks error. ");
633 
634 #if STK_GEOMDECOMP_DEBUG>=2
635  {
636  debug_print_decomp_export( zoltan, m_zoltan_id_ );
637  }
638 #endif
639  return;
640 
641 }
642 
643 double Zoltan::init_zoltan_library () {
644  float version = 0.0;
645  if ( Zoltan_Initialize( 0, NULL, &version) != ZOLTAN_OK )
646  throw std::runtime_error("Return code from Zoltan_Initialize() != ZOLTAN_OK ");
647 
648  static_zoltan_version(version);
649  std::ostringstream s;
650  s << version;
651 
652  //sierra::ProductRegistry::instance().addTPL("Zoltan", s.str());
653 
654  return version;
655 }
656 
657 
658 
659 //: ==========
660 //: Destructor
661 //: ==========
662 
663 Zoltan::~Zoltan()
664 {
665  if ( m_zoltan_id_ != NULL ) {
666  Zoltan_Destroy( &m_zoltan_id_ );
667  }
668  m_zoltan_id_ = NULL ;
669  if(Name_Conversion) {
670  delete Name_Conversion;
671  Name_Conversion = NULL;
672  }
673  if(Value_Conversion) {
674  delete Value_Conversion;
675  Value_Conversion = NULL;
676  }
677 }
678 
679 void
680 Zoltan::reset_dest_proc_data()
681 {
682  const int proc = parallel_machine_rank(comm_);
683  const unsigned size = m_mesh_information_.mesh_entities.size();
684  m_mesh_information_.dest_proc_ids.assign(size, proc);
685 }
686 
687 void
688 Zoltan::set_destination_proc(const unsigned moid,
689  const unsigned proc )
690 {
691  m_mesh_information_.dest_proc_ids[ moid ] = proc;
692 }
693 
694 unsigned
695 Zoltan::destination_proc(const unsigned moid) const
696 {
697  return m_mesh_information_.dest_proc_ids[ moid ];
698 }
699 
700 bool
701 Zoltan::find_mesh_entity(const mesh::Entity * entity, unsigned & moid) const
702 {
703  unsigned len = m_mesh_information_.mesh_entities.size();
704  for(moid = 0; moid < len; ++moid)
705  {
706  if(m_mesh_information_.mesh_entities[moid] == entity) return true;
707  }
708  return false;
709 }
710 
711 int
712 Zoltan::get_new_partition(stk_classic::mesh::EntityProcVec &rebal_spec)
713 {
714  const unsigned entity_iter_len = m_mesh_information_.mesh_entities.size();
715  for (unsigned entity_iter =0; entity_iter != entity_iter_len; ++entity_iter) {
716  mesh::Entity * mesh_ent = mesh_entity(entity_iter);
717  int proc = destination_proc(entity_iter);
718  mesh::EntityProc et(mesh_ent, proc);
719  rebal_spec.push_back(et);
720  }
721  return 0;
722 }
723 
724 const VectorField *
725 Zoltan::entity_coord_ref() const
726 {
727  return m_mesh_information_.nodal_coord_ref;
728 }
729 
730 mesh::Entity *
731 Zoltan::mesh_entity(const unsigned moid ) const
732 {
733  return m_mesh_information_.mesh_entities[ moid ];
734 }
735 
736 double
737 Zoltan::entity_weight(const unsigned moid ) const
738 {
739  double mo_weight = 1.0;
740  if (entity_weight_ref()) {
741  mo_weight = * static_cast<double *>
742  ( mesh::field_data (*entity_weight_ref(), *m_mesh_information_.mesh_entities[ moid ]));
743  }
744  return mo_weight;
745 }
746 
747 const ScalarField *
748 Zoltan::entity_weight_ref() const
749 {
750  return m_mesh_information_.elem_weight_ref;
751 }
752 
753 unsigned
754 Zoltan::num_moid() const
755 {
756  return m_mesh_information_.mesh_entities.size() ;
757 }
758 
759 const std::string &Zoltan::parameter_entry_name() const
760 {
761  return m_parameter_entry_name_;
762 }
763 
764 
765 //: Load Balance calls or Load "Re-partitioning" calls
766 
767 int Zoltan::register_callbacks()
768 {
769  /*
770  * Register the Zoltan/SIERRA "call-back" (querry) functions.
771  * Use ONLY THE BARE ESSENTIALS for decompositions:
772  *
773  * Zoltan_Set_Num_Obj_Fn
774  * Zoltan_Set_Obj_List_Fn
775  * Zoltan_Set_Num_Geom_Fn
776  * Zoltan_Set_Geom_Fn
777  */
778 
779  /*
780  * flag for what data is to be registered with the zoltan callback
781  * functions in combination with the "static_cast<CLASS>(data)"
782  * statement used in the zoltan interface routine,
783  * Fmwk_Zoltaninterface.C
784  *
785  */
786 
787  if ( Zoltan_Set_Num_Obj_Fn( m_zoltan_id_,
788  stkCallback_Num_Elements,
789  this )
790  != ZOLTAN_OK ) {
791  throw std::runtime_error("Zoltan_Set_Num_Obj_Fn using stkCallback_Num_Elements failed to register");
792  }
793  if ( Zoltan_Set_Obj_List_Fn( m_zoltan_id_, stkCallback_Element_List,this )
794  != ZOLTAN_OK ) {
795  throw std::runtime_error("Zoltan_Set_Obj_List_Fn using stkCallback_Element_List");
796  }
797  if ( Zoltan_Set_Num_Geom_Fn( m_zoltan_id_, stkCallback_Num_Dimensions,this )
798  != ZOLTAN_OK ) {
799  throw std::runtime_error("Zoltan_Set_Num_Geom_Fn using stkCallback_Num_Dimensions");
800  }
801  if ( Zoltan_Set_Geom_Fn( m_zoltan_id_, stkCallback_Centroid_Coord,this )
802  != ZOLTAN_OK ) {
803  throw std::runtime_error("Zoltan_Set_Geom_Fn using stkCallback_Centroid_Coord");
804  }
805  if ( Zoltan_Set_Num_Edges_Fn( m_zoltan_id_, stkCallback_Num_Edges,this )
806  != ZOLTAN_OK ) {
807  throw std::runtime_error("Zoltan_Set_Num_Edges_Fn using stkCallback_Num_Edges");
808  }
809  if ( Zoltan_Set_Edge_List_Fn( m_zoltan_id_, stkCallback_Edge_List,this )
810  != ZOLTAN_OK ) {
811  throw std::runtime_error("Zoltan_Set_Edge_List_Fn using stkCallback_Edge_List");
812  }
813 
814  return 0;
815 
816 }
817 
818 
819 int Zoltan::evaluate( int print_stats,
820  int* nentity,
821  double* entity_wgt,
822  int* ncuts,
823  double* cut_wgt,
824  int* nboundary,
825  int* nadj )
826 {
827  int ierr = 0;
828 
829  ZOLTAN_BALANCE_EVAL eval = {0};
830  ZOLTAN_GRAPH_EVAL graph = {{0}};
831  if (Zoltan_LB_Eval_Balance( m_zoltan_id_, print_stats, &eval)) ierr = 1;
832  if (Zoltan_LB_Eval_Graph( m_zoltan_id_, print_stats, &graph) ) ierr = 1;
833  *nentity = (int)eval.nobj[0];
834  *entity_wgt = eval.obj_wgt[0];
835  *ncuts = (int)graph.cuts[0];
836  *cut_wgt = graph.cut_wgt[0];
837  *nboundary = (int)graph.num_boundary[0];
838  *nadj = (int)graph.nnborparts[0];
839 
840  return ierr;
841 
842 }
843 
844 void Zoltan::determine_new_partition (bool &RebalancingNeeded)
845 {
846  //: Transfer export global ID lists and owning processors
847  //: to SIERRA Framework's data structures
848 
855  int length_gid, length_lid;
856 
857  int new_decomp;
858  int num_imported;
859  ZOLTAN_ID_PTR import_gids;
860  ZOLTAN_ID_PTR import_lids;
861  int *import_procs=NULL;
862  int num_exported;
863  ZOLTAN_ID_PTR export_gids;
864  ZOLTAN_ID_PTR export_lids;
865  int *export_procs=NULL;
866 
867 
868 
924  int status = Zoltan_LB_Balance( m_zoltan_id_, &new_decomp,
925  &length_gid , &length_lid,
926  &num_imported, &import_gids,
927  &import_lids, &import_procs,
928  &num_exported, &export_gids,
929  &export_lids, &export_procs );
930  if (status != ZOLTAN_OK) {
931  throw std::runtime_error("Zoltan_Balance() returned error code " + status);
932  }
933 
934  //: Initialize destination processor IDs (dest_proc_ids)
935  reset_dest_proc_data();
936 
937  int actual_exported = 0;
938  if ( new_decomp && ( num_exported != -1 ) ) {
939  const unsigned parallel_rank = parallel_machine_rank(comm_);
940  /* New Decomposition was generated */
941  for (int j=0; j < num_exported; ++j ) {
942 
943  //: Get exported region, local, global, and processor ids
944  //const unsigned rid = export_gids[ j*::num_gid_entries() ]; // Region ID variable
945  //if (!rid) throw runtime_error ("Region ID variable should be non-zero.");
946  const unsigned lid = export_lids[ j*::num_lid_entries() ]; // Local ID variable
947  const unsigned pid = export_procs[ j ]; // Exported Processor ID (i.e., MPI "rank" )
948 
949  if (parallel_rank != pid) {
950  ++actual_exported;
951  set_destination_proc(lid, pid);
952  }
953  }
954  }
955 
956  RebalancingNeeded = 0 ;
957  if (new_decomp) {
958  int rebalneeded=0;
959  stk_classic::all_reduce_sum(comm_, &actual_exported, &rebalneeded, 1);
960  if (rebalneeded) RebalancingNeeded = 1;
961  }
962 
966  if ( ZOLTAN_OK !=
967  Zoltan_LB_Free_Data( &import_gids, &import_lids, &import_procs,
968  &export_gids, &export_lids, &export_procs )) {
969  throw runtime_error (" FATAL ERROR in Zoltan_LB_Free_Data.");
970  }
971 
972 }
973 
974 void Zoltan::convert_names_and_values(const Parameters &from, Parameters &to)
975 {
976  /* First time through, fill the conversion tables. */
977  if (!Name_Conversion) {
978  Name_Conversion = new Parameters;
979  fill_name_conversion (*Name_Conversion);
980  }
981  if (!Value_Conversion) {
982  Value_Conversion = new Parameters;
983  fill_value_conversion(*Value_Conversion);
984  }
985 
986  // NOTE: "LOAD BALANCING METHOD" default
987  // is also hard coded in fill_default_values();
988  std::string algorithm;
989  const std::string keyname("LOAD BALANCING METHOD");
990  if( from.isParameter(keyname) )
991  algorithm = from.get<std::string>(keyname);
992  else
993  algorithm = "0";
994 
995  const Parameters & General = Name_Conversion->sublist("General");
996  const Parameters & Algorithm = Name_Conversion->sublist(algorithm);
997 
998  Parameters::ConstIterator
999  from_iter = from.begin(),
1000  from_end = from.end();
1001 
1002  /* Iterate over all of the input parameters to find proper
1003  Zoltan names and Zoltan parameter values. */
1004  for (; from_iter != from_end; ++from_iter) {
1005  std::string
1006  from_name = from.name(from_iter),
1007  to_name;
1008 
1009  /* Check to see if this is a general parameter name
1010  and if not, check if it is an algorithm specific name. */
1011 
1012  if (General.isParameter(from_name)) {
1013  to_name = General.get<std::string>(from_name);
1014  } else {
1015  to_name = Algorithm.get<std::string>(from_name);
1016  }
1017 
1018  /* Now convert the parameter value to the correct form.
1019  Only a couple of parameters have parameter conversion.
1020  The ones converted are nested in Value_Conversion.
1021  */
1022  std::string to_value = Teuchos::getValue<string>(from.entry(from_iter));
1023  //if (Value_Conversion->isParameter(from_name)) to_value = Value_Conversion->get<std::string>(to_value);
1024  if (Value_Conversion->isParameter(from_name)) to_value = Value_Conversion->sublist(from_name).get<std::string>(to_value);
1025  if (!to_name.empty()) to.set(to_name, to_value);
1026  }
1027 }
1028 
1029 void Zoltan::merge_default_values(const Parameters &from,
1030  Parameters &to)
1031 {
1032  Parameters default_values;
1033  fill_default_values(default_values);
1034  to.setParameters(default_values);
1035  to.setParameters(from);
1036 }
std::ostream & output()
Function output returns the processor output log stream. This stream is connected via an mpi_filebuf ...
Definition: Env.cpp:264
FEMMetaData is a class that implements a Finite Element Method skin on top of the Sierra Tool Kit Met...
Definition: FEMMetaData.hpp:54
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
Definition: FieldData.hpp:116
void all_reduce_sum(ParallelMachine comm, const double *local, double *global, unsigned count)
Parallel summation to all processors.
int pid()
Function pid returns the process id of the process running the application.
Definition: Platform.cpp:557
EntityRank element_rank() const
Returns the element rank which is always equal to spatial dimension.
const EntityKey & key() const
The globally unique key ( entity type + identifier ) of this entity.
Definition: Entity.hpp:138
std::pair< Entity *, unsigned > EntityProc
Pairing of an entity with a processor rank.
Definition: Types.hpp:111
int parallel_rank()
function parallel_rank returns the rank of this processor in the current mpi communicator.
Definition: Env.cpp:318
Field with defined data type and multi-dimensions (if any)
Definition: Field.hpp:118
unsigned parallel_machine_rank(ParallelMachine parallel_machine)
Member function parallel_machine_rank ...
Definition: Parallel.cpp:29
PairIterRelation relations() const
All Entity relations for which this entity is a member. The relations are ordered from lowest entity-...
Definition: Entity.hpp:161
A fundamental unit within the discretization of a problem domain, including but not limited to nodes...
Definition: Entity.hpp:120
Sierra Toolkit.
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
virtual unsigned num_elems() const =0
Return the total number of mesh entities in all lists.
Class for determining the optimal partitioning of mesh entities.
Definition: GeomDecomp.hpp:71
EntityRank node_rank() const
Returns the node rank, which is always zero.
static FEMMetaData & get(const MetaData &meta)
Getter for FEMMetaData off of a MetaData object.
void output_flush()
Function output_flush flushes all output on all currently open synchronous outptu files which were op...
Definition: Env.cpp:1105
unsigned owner_rank() const
Parallel processor rank of the processor which owns this entity.
Definition: Entity.hpp:175