Sierra Toolkit  Version of the Day
ParallelComm.hpp
1 /*------------------------------------------------------------------------*/
2 /* Copyright 2010 Sandia Corporation. */
3 /* Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive */
4 /* license for use of this work by or on behalf of the U.S. Government. */
5 /* Export of this program may require a license from the */
6 /* United States Government. */
7 /*------------------------------------------------------------------------*/
8 
9 #ifndef stk_util_parallel_ParallelComm_hpp
10 #define stk_util_parallel_ParallelComm_hpp
11 
12 #include <cstddef>
13 #include <iosfwd>
14 #include <stk_util/parallel/Parallel.hpp>
15 
16 //------------------------------------------------------------------------
17 
18 namespace stk_classic {
19 
28 class CommAll ;
29 
32 class CommBuffer ;
33 
47  const unsigned num_msg_bound ,
48  unsigned & num_msg_maximum ,
49  const unsigned * const send_size ,
50  unsigned * const recv_size ,
51  bool local_flag = false );
52 
56  const unsigned * const send_size ,
57  unsigned * const recv_size ,
58  bool local_flag = false );
59 
60 //------------------------------------------------------------------------
61 
62 class CommBuffer {
63 public:
64 
66  template<typename T> CommBuffer &pack( const T & value );
67 
69  template<typename T> CommBuffer &pack( const T * value , size_t number );
70 
72  template<typename T> CommBuffer &unpack( T & value );
73 
75  template<typename T> CommBuffer &unpack( T * value , size_t number );
76 
78  template<typename T> CommBuffer &peek( T & value );
79 
81  template<typename T> CommBuffer &peek( T * value , size_t number );
82 
84  template<typename T> CommBuffer &skip( size_t number );
85 
87  void reset();
88 
92  size_t capacity() const ;
93 
98  size_t size() const ;
99 
106  ptrdiff_t remaining() const ;
107 
109  void * buffer() const ;
110 
111  ~CommBuffer();
112  CommBuffer();
113 
114 private:
115  friend class CommAll ;
116  friend class CommGather ;
117  friend class CommBroadcast ;
118 
119  static CommBuffer * allocate( const unsigned, const unsigned * const );
120  static void deallocate( const unsigned , CommBuffer * );
121 
122  void pack_overflow() const ;
123  void unpack_overflow() const ;
124 
125  CommBuffer( const CommBuffer & );
126  CommBuffer & operator = ( const CommBuffer & );
127 
128  typedef unsigned char * ucharp ;
129 
130  ucharp m_beg ;
131  ucharp m_ptr ;
132  ucharp m_end ;
133 };
134 
135 //------------------------------------------------------------------------
136 
137 class CommAll {
138 public:
139 
140  ParallelMachine parallel() const { return m_comm ; }
141  unsigned parallel_size() const { return m_size ; }
142  unsigned parallel_rank() const { return m_rank ; }
143 
145  CommBuffer & send_buffer( unsigned ) const ;
146 
148  CommBuffer & recv_buffer( unsigned ) const ;
149 
150  //----------------------------------------
154  CommAll();
155 
163  bool allocate_buffers( ParallelMachine ,
164  const unsigned num_msg_bounds ,
165  const unsigned * const send_size ,
166  const unsigned * const recv_size ,
167  const bool local_flag = false );
168 
169  //----------------------------------------
182  explicit CommAll( ParallelMachine );
183 
190  bool allocate_buffers( const unsigned num_msg_bounds ,
191  const bool symmetric = false ,
192  const bool local_flag = false );
193 
194  //----------------------------------------
196  void communicate();
197 
198  //----------------------------------------
200  void swap_send_recv();
201 
205  void reset_buffers();
206 
207  ~CommAll();
208 
209 private:
210 
211  CommAll( const CommAll & );
212  CommAll & operator = ( const CommAll & );
213 
214  void rank_error( const char * , unsigned ) const ;
215 
216  bool allocate_buffers( const unsigned * const send_size ,
217  const unsigned * const recv_size ,
218  bool local_flag );
219 
220  ParallelMachine m_comm ;
221  unsigned m_size ;
222  unsigned m_rank ;
223  unsigned m_bound ;
224  unsigned m_max ;
225  CommBuffer * m_send ;
226  CommBuffer * m_recv ;
227 };
228 
229 //------------------------------------------------------------------------
230 
231 class CommBroadcast {
232 public:
233 
234  ParallelMachine parallel() const { return m_comm ; }
235  unsigned parallel_size() const { return m_size ; }
236  unsigned parallel_rank() const { return m_rank ; }
237 
239  CommBuffer & send_buffer();
240 
242  CommBuffer & recv_buffer();
243 
244  //----------------------------------------
245 
246  CommBroadcast( ParallelMachine , unsigned root_rank );
247 
248  void communicate();
249 
250  bool allocate_buffer( const bool local_flag = false );
251 
252  ~CommBroadcast();
253 
254 private:
255 
256  CommBroadcast();
257  CommBroadcast( const CommBroadcast & );
258  CommBroadcast & operator = ( const CommBroadcast & );
259 
260  ParallelMachine m_comm ;
261  unsigned m_size ;
262  unsigned m_rank ;
263  unsigned m_root_rank ;
264  CommBuffer m_buffer ;
265 };
266 
267 //----------------------------------------------------------------------
268 
269 class CommGather {
270 public:
271 
272  ParallelMachine parallel() const { return m_comm ; }
273  unsigned parallel_size() const { return m_size ; }
274  unsigned parallel_rank() const { return m_rank ; }
275 
276  ~CommGather();
277 
278  CommGather( ParallelMachine , unsigned root_rank , unsigned send_size );
279 
280  CommBuffer & send_buffer() { return m_send ; }
281 
282  void communicate();
283 
284  CommBuffer & recv_buffer( unsigned );
285 
286  void reset();
287 
288 private:
289 
290  CommGather();
291  CommGather( const CommBroadcast & );
292  CommGather & operator = ( const CommBroadcast & );
293 
294  ParallelMachine m_comm ;
295  unsigned m_size ;
296  unsigned m_rank ;
297  unsigned m_root_rank ;
298  CommBuffer m_send ;
299  CommBuffer * m_recv ;
300  int * m_recv_count ;
301  int * m_recv_displ ;
302 };
303 
304 }
305 
306 //----------------------------------------------------------------------
307 //----------------------------------------------------------------------
308 // Inlined template implementations for the CommBuffer
309 
310 namespace stk_classic {
311 
312 template<unsigned N> struct CommBufferAlign ;
313 
314 template<>
315 struct CommBufferAlign<1> {
316  static size_t align( size_t ) { return 0 ; }
317 };
318 
319 template<unsigned N>
320 struct CommBufferAlign {
321  static size_t align( size_t i ) { i %= N ; return i ? ( N - i ) : 0 ; }
322 };
323 
324 template<typename T>
325 inline
326 CommBuffer &CommBuffer::pack( const T & value )
327 {
328  enum { Size = sizeof(T) };
329  size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
330  if ( m_beg ) {
331  if ( m_end < m_ptr + nalign + Size ) { pack_overflow(); }
332  while ( nalign ) { --nalign ; *m_ptr = 0 ; ++m_ptr ; }
333  T * tmp = reinterpret_cast<T*>(m_ptr);
334  *tmp = value ;
335  m_ptr = reinterpret_cast<ucharp>( ++tmp );
336  }
337  else {
338  m_ptr += nalign + Size ;
339  }
340  return *this;
341 }
342 
343 template<typename T>
344 inline
345 CommBuffer &CommBuffer::pack( const T * value , size_t number )
346 {
347  enum { Size = sizeof(T) };
348  size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
349  if ( m_beg ) {
350  if ( m_end < m_ptr + nalign + number * Size ) { pack_overflow(); }
351  while ( nalign ) { --nalign ; *m_ptr = 0 ; ++m_ptr ; }
352  T * tmp = reinterpret_cast<T*>(m_ptr);
353  while ( number ) { --number ; *tmp = *value ; ++tmp ; ++value ; }
354  m_ptr = reinterpret_cast<ucharp>( tmp );
355  }
356  else {
357  m_ptr += nalign + number * Size ;
358  }
359  return *this;
360 }
361 
362 template<typename T>
363 inline
364 CommBuffer &CommBuffer::skip( size_t number )
365 {
366  enum { Size = sizeof(T) };
367  m_ptr += CommBufferAlign<Size>::align( m_ptr - m_beg ) + Size * number ;
368  if ( m_beg && m_end < m_ptr ) { unpack_overflow(); }
369  return *this;
370 }
371 
372 template<typename T>
373 inline
374 CommBuffer &CommBuffer::unpack( T & value )
375 {
376  enum { Size = sizeof(T) };
377  const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
378  T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
379  value = *tmp ;
380  m_ptr = reinterpret_cast<ucharp>( ++tmp );
381  if ( m_end < m_ptr ) { unpack_overflow(); }
382  return *this;
383 }
384 
385 template<typename T>
386 inline
387 CommBuffer &CommBuffer::unpack( T * value , size_t number )
388 {
389  enum { Size = sizeof(T) };
390  const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
391  T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
392  while ( number ) { --number ; *value = *tmp ; ++tmp ; ++value ; }
393  m_ptr = reinterpret_cast<ucharp>( tmp );
394  if ( m_end < m_ptr ) { unpack_overflow(); }
395  return *this;
396 }
397 
398 template<typename T>
399 inline
400 CommBuffer &CommBuffer::peek( T & value )
401 {
402  enum { Size = sizeof(T) };
403  const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
404  T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
405  value = *tmp ;
406  if ( m_end < reinterpret_cast<ucharp>(++tmp) ) { unpack_overflow(); }
407  return *this;
408 }
409 
410 template<typename T>
411 inline
412 CommBuffer &CommBuffer::peek( T * value , size_t number )
413 {
414  enum { Size = sizeof(T) };
415  const size_t nalign = CommBufferAlign<Size>::align( m_ptr - m_beg );
416  T * tmp = reinterpret_cast<T*>( m_ptr + nalign );
417  while ( number ) { --number ; *value = *tmp ; ++tmp ; ++value ; }
418  if ( m_end < reinterpret_cast<ucharp>(tmp) ) { unpack_overflow(); }
419  return *this;
420 }
421 
422 inline
423 void CommBuffer::reset()
424 { m_ptr = m_beg ; }
425 
426 inline
427 size_t CommBuffer::capacity() const
428 { return m_end - m_beg ; }
429 
430 inline
431 size_t CommBuffer::size() const
432 { return m_ptr - m_beg ; }
433 
434 inline
435 ptrdiff_t CommBuffer::remaining() const
436 { return m_end - m_ptr ; }
437 
438 inline
439 void * CommBuffer::buffer() const
440 { return static_cast<void*>( m_beg ); }
441 
442 //----------------------------------------------------------------------
443 // Inline implementations for the CommAll
444 
445 inline
446 CommBuffer & CommAll::send_buffer( unsigned p ) const
447 {
448  if ( m_size <= p ) { rank_error("send_buffer",p); }
449  return m_send[p] ;
450 }
451 
452 inline
453 CommBuffer & CommAll::recv_buffer( unsigned p ) const
454 {
455  if ( m_size <= p ) { rank_error("recv_buffer",p); }
456  return m_recv[p] ;
457 }
458 
459 }
460 
461 //----------------------------------------------------------------------
462 //----------------------------------------------------------------------
463 
464 #endif
465 
bool comm_dense_sizes(ParallelMachine comm, const unsigned *const send_size, unsigned *const recv_size, bool local_flag)
int parallel_rank()
function parallel_rank returns the rank of this processor in the current mpi communicator.
Definition: Env.cpp:318
bool comm_sizes(ParallelMachine comm, const unsigned num_msg_bound, unsigned &num_msg_maximum, const unsigned *const send_size, unsigned *const recv_size, bool local_flag)
Sierra Toolkit.
MPI_Comm ParallelMachine
Definition: Parallel.hpp:32
void reset(MPI_Comm new_comm)
Function reset determines new parallel_size and parallel_rank. Flushes, closes, and reopens log files...
Definition: Env.cpp:1067
int parallel_size()
function parallel_size returns the number of processors in the current mpi communicator.
Definition: Env.cpp:314