14 #include <stk_util/parallel/ParallelComm.hpp> 15 #include <stk_util/parallel/ParallelReduce.hpp> 17 #include <stk_mesh/base/BulkData.hpp> 18 #include <stk_mesh/base/FieldData.hpp> 19 #include <stk_mesh/base/FieldParallel.hpp> 25 void communicate_field_data(
26 const Ghosting & ghosts ,
27 const std::vector< const FieldBase *> & fields )
29 if ( fields.empty() ) {
return; }
31 const BulkData & mesh = BulkData::get(ghosts);
35 const std::vector<const FieldBase *>::const_iterator fe = fields.end();
36 const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
37 std::vector<const FieldBase *>::const_iterator fi ;
41 const unsigned zero = 0 ;
45 for ( std::vector<Entity*>::const_iterator
46 i = mesh.entity_comm().begin() ;
47 i != mesh.entity_comm().end() ; ++i ) {
52 for ( fi = fb ; fi != fe ; ++fi ) {
53 const FieldBase & f = **fi ;
58 if ( ghosts.ordinal() == ec->ghost_id ) {
60 send_size[ ec->proc ] += e_size ;
63 recv_size[ ec->proc ] += e_size ;
74 const unsigned *
const s_size = & send_size[0] ;
75 const unsigned *
const r_size = & recv_size[0] ;
76 sparse.allocate_buffers( mesh.parallel(),
parallel_size / 4 , s_size, r_size);
81 for ( std::vector<Entity*>::const_iterator
82 i = mesh.entity_comm().begin() ;
83 i != mesh.entity_comm().end() ; ++i ) {
87 for ( fi = fb ; fi != fe ; ++fi ) {
88 const FieldBase & f = **fi ;
93 reinterpret_cast<unsigned char *
>(
field_data( f , e ));
97 if ( ghosts.ordinal() == ec->ghost_id ) {
98 CommBuffer & b = sparse.send_buffer( ec->proc );
99 b.pack<
unsigned char>( ptr , size );
109 sparse.communicate();
113 for ( std::vector<Entity*>::const_iterator
114 i = mesh.entity_comm().begin() ;
115 i != mesh.entity_comm().end() ; ++i ) {
119 for ( fi = fb ; fi != fe ; ++fi ) {
120 const FieldBase & f = **fi ;
124 unsigned char * ptr =
125 reinterpret_cast<unsigned char *
>(
field_data( f , e ));
129 if ( ghosts.ordinal() == ec->ghost_id ) {
130 CommBuffer & b = sparse.recv_buffer( ec->proc );
131 b.unpack<
unsigned char>( ptr , size );
142 void communicate_field_data(
144 const std::vector<EntityProc> & domain ,
145 const std::vector<EntityProc> & range ,
146 const std::vector<const FieldBase *> & fields)
148 if ( fields.empty() ) {
return; }
152 const bool asymmetric = & domain != & range ;
154 const std::vector<const FieldBase *>::const_iterator fe = fields.end();
155 const std::vector<const FieldBase *>::const_iterator fb = fields.begin();
156 std::vector<const FieldBase *>::const_iterator fi ;
160 const unsigned zero = 0 ;
164 std::vector<EntityProc>::const_iterator i ;
166 for ( i = domain.begin() ; i != domain.end() ; ++i ) {
167 Entity & e = * i->first ;
168 const unsigned p = i->second ;
171 unsigned e_size = 0 ;
172 for ( fi = fb ; fi != fe ; ++fi ) {
173 const FieldBase & f = **fi ;
176 send_size[ p ] += e_size ;
180 for ( i = range.begin() ; i != range.end() ; ++i ) {
181 Entity & e = * i->first ;
182 const unsigned p = i->second ;
184 if ( asymmetric || p == e.owner_rank() ) {
185 unsigned e_size = 0 ;
186 for ( fi = fb ; fi != fe ; ++fi ) {
187 const FieldBase & f = **fi ;
190 recv_size[ p ] += e_size ;
199 const unsigned *
const s_size = & send_size[0] ;
200 const unsigned *
const r_size = & recv_size[0] ;
201 sparse.allocate_buffers( machine,
parallel_size / 4 , s_size, r_size);
206 for ( i = domain.begin() ; i != domain.end() ; ++i ) {
207 Entity & e = * i->first ;
208 const unsigned p = i->second ;
211 CommBuffer & b = sparse.send_buffer( p );
212 for ( fi = fb ; fi != fe ; ++fi ) {
213 const FieldBase & f = **fi ;
216 unsigned char * ptr =
reinterpret_cast<unsigned char *
>(
field_data( f , e ));
217 b.pack<
unsigned char>( ptr , size );
225 sparse.communicate();
229 for ( i = range.begin() ; i != range.end() ; ++i ) {
230 Entity & e = * i->first ;
231 const unsigned p = i->second ;
233 if ( asymmetric || p == e.owner_rank() ) {
234 CommBuffer & b = sparse.recv_buffer( p );
235 for ( fi = fb ; fi != fe ; ++fi ) {
236 const FieldBase & f = **fi ;
239 unsigned char * ptr =
reinterpret_cast<unsigned char *
>(
field_data( f , e ));
240 b.unpack<
unsigned char>( ptr , size );
249 void communicate_field_data(
250 const BulkData & mesh ,
251 const unsigned field_count ,
252 const FieldBase * fields[] ,
255 const std::vector<Entity*> & entity_comm = mesh.entity_comm();
261 const unsigned zero = 0 ;
266 for ( j = 0 ; j < field_count ; ++j ) {
267 const FieldBase & f = * fields[j] ;
268 for ( std::vector<Entity*>::const_iterator
269 i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
274 ec = e.comm() ; ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
275 msg_size[ ec->proc ] += size ;
284 const unsigned *
const s_size = & msg_size[0] ;
285 sparse.allocate_buffers( mesh.parallel(),
parallel_size / 4 , s_size, s_size);
290 for ( j = 0 ; j < field_count ; ++j ) {
291 const FieldBase & f = * fields[j] ;
292 for ( std::vector<Entity*>::const_iterator
293 i = entity_comm.begin() ; i != entity_comm.end() ; ++i ) {
297 unsigned char * ptr =
298 reinterpret_cast<unsigned char *
>(
field_data( f , e ));
300 ec = e.comm() ; ! ec.empty() && ec->ghost_id == 0 ; ++ec ) {
301 CommBuffer & b = sparse.send_buffer( ec->proc );
302 b.pack<
unsigned char>( ptr , size );
310 sparse.communicate();
313 void communicate_field_data_verify_read( CommAll & sparse )
315 std::ostringstream msg ;
317 for (
unsigned p = 0 ; p < sparse.parallel_size() ; ++p ) {
318 if ( sparse.recv_buffer( p ).remaining() ) {
319 msg <<
"P" << sparse.parallel_rank()
320 <<
" Unread data from P" << p << std::endl ;
324 all_reduce( sparse.parallel() , ReduceSum<1>( & error ) );
325 ThrowErrorMsgIf( error, msg.str() );
unsigned field_data_size(const FieldBase &f, const Bucket &k)
Size, in bytes, of the field data for each entity.
FieldTraits< field_type >::data_type * field_data(const field_type &f, const Bucket::iterator i)
Pointer to the field data array.
int parallel_rank()
function parallel_rank returns the rank of this processor in the current mpi communicator.
unsigned parallel_machine_rank(ParallelMachine parallel_machine)
Member function parallel_machine_rank ...
unsigned parallel_machine_size(ParallelMachine parallel_machine)
Member function parallel_machine_size ...
int parallel_size()
function parallel_size returns the number of processors in the current mpi communicator.
PairIter< std::vector< EntityCommInfo >::const_iterator > PairIterEntityComm
Span of ( communication-subset-ordinal , process-rank ) pairs for the communication of an entity...