ESyS-Particle  4.0.1
LatticeMaster.hpp
1 
2 // //
3 // Copyright (c) 2003-2011 by The University of Queensland //
4 // Earth Systems Science Computational Centre (ESSCC) //
5 // http://www.uq.edu.au/esscc //
6 // //
7 // Primary Business: Brisbane, Queensland, Australia //
8 // Licensed under the Open Software License version 3.0 //
9 // http://www.opensource.org/licenses/osl-3.0.php //
10 // //
12 
13 #include "Parallel/GetRef_cmd.h"
14 #include <map>
15 #include <set>
16 
17 template <typename TmplVisitor>
18 void CLatticeMaster::visitMeshFaceReferences(const string &meshName)
19 {
20  throw std::runtime_error("CLatticeMaster::visitMeshFaceReferences: Not implemented.");
21 }
22 
23 template <typename TmplVisitor>
24 void CLatticeMaster::visitMesh2dNodeReferences(const string &meshName, TmplVisitor &visitor)
25 {
26  console.XDebug()<<"CLatticeMaster::visitMesh2dNodeReferences( " << meshName << ")\n";
27  GetNodeRefCommand cmd(getGlobalRankAndComm(),meshName);
28  // broadcast command
29  cmd.broadcast();
30 
31  // receive data (multimap)
32  std::multimap<int,int> ref_mmap;
33  m_tml_global_comm.gather(ref_mmap);
34  // collate into set
35  std::set<int> ref_set; //== this is the set of node ids ==
36  for(std::multimap<int,int>::iterator iter=ref_mmap.begin();
37  iter!=ref_mmap.end();
38  iter++){
39  ref_set.insert(iter->second);
40  }
41  for (std::set<int>::const_iterator it = ref_set.begin(); it != ref_set.end(); it++)
42  {
43  visitor.visitNodeRef(*it);
44  }
45  console.XDebug()<<"end CLatticeMaster::visitMesh2dNodeReferences()\n";
46 }
47 
48 template <typename TmplVisitor>
49 void CLatticeMaster::visitMesh2dEdgeStress(const string &meshName, TmplVisitor &visitor)
50 {
51  console.XDebug()<<"CLatticeMaster::visitMesh2dEdgeStress( " << meshName << ")\n";
52  std::multimap<int,pair<int,Vec3> > temp_mm;
53  std::map<int,Vec3> data; //=== map of id, value ===
54 
55  BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETMESH2DSTRESS);
56  cmd.append(meshName.c_str());
57  cmd.broadcastCommand();
58  cmd.broadcastBuffer();
59 
60  // get data from slaves
61  m_tml_global_comm.gather(temp_mm);
62 
63  // add data together
64  for(std::multimap<int,pair<int,Vec3> >::iterator iter=temp_mm.begin();
65  iter!=temp_mm.end();
66  iter++){
67  if(data.find((iter->second).first)==data.end()){ // id not in data -> insert
68  data.insert(iter->second);
69  } else { // id is in data -> add
70  data[(iter->second).first]+=(iter->second).second;
71  }
72  }
73 
74  for (std::map<int,Vec3>::const_iterator it = data.begin(); it != data.end(); it++)
75  {
76  visitor.visitRefStressPair(it->first, it->second);
77  }
78 
79  cmd.wait("visitMesh2dEdgeStress");
80  console.XDebug()<<"end CLatticeMaster::visitMesh2dEdgeStress()\n";
81 }
82 
83 template <typename TmplVisitor>
84 void CLatticeMaster::visitTriMeshFaceForce(
85  const string &meshName,
86  TmplVisitor &visitor
87 )
88 {
89  console.XDebug()<<"CLatticeMaster::visitTriMeshFaceForce( " << meshName << ")\n";
90  std::multimap<int,pair<int,Vec3> > temp_mm;
91  std::map<int,Vec3> data; //=== map of id, value ===
92 
93  BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETTRIMESHFORCE);
94  cmd.append(meshName.c_str());
95  cmd.broadcastCommand();
96  cmd.broadcastBuffer();
97 
98  // get data from slaves
99  m_tml_global_comm.gather(temp_mm);
100 
101  // add data together
102  for(std::multimap<int,pair<int,Vec3> >::iterator iter=temp_mm.begin();
103  iter!=temp_mm.end();
104  iter++){
105  if(data.find((iter->second).first)==data.end()){ // id not in data -> insert
106  data.insert(iter->second);
107  } else { // id is in data -> add
108  data[(iter->second).first]+=(iter->second).second;
109  }
110  }
111 
112  for (std::map<int,Vec3>::const_iterator it = data.begin(); it != data.end(); it++)
113  {
114  visitor.visitRefForcePair(it->first, it->second);
115  }
116 
117  cmd.wait("visitTriMeshFaceStress");
118  console.XDebug()<<"end CLatticeMaster::visitTriMeshFaceForce()\n";
119 }
120 
121 template <typename TmplVisitor, typename TmplParticle>
122 void CLatticeMaster::visitParticlesOfType(
123  const IdVector &particleIdVector,
124  TmplVisitor &visitor
125 )
126 {
127  console.Debug() << "CLatticeMaster::visitParticlesOfType: enter\n";
128  typedef std::multimap<int,TmplParticle> ParticleMMap;
129  ParticleMMap particleMMap;
130 
131  console.Debug()
132  << "CLatticeMaster::visitParticlesOfType: broadcasting command\n";
133  BroadcastCommand cmd(getGlobalRankAndComm(), CMD_GETIDPARTICLEDATA);
134  cmd.broadcastCommand();
135 
136  console.Debug()
137  << "CLatticeMaster::visitParticlesOfType: broadcasting particle id's\n";
138  m_tml_global_comm.broadcast_cont(particleIdVector);
139 
140  console.Debug()
141  << "CLatticeMaster::visitParticlesOfType:"
142  << " gathering particle data from workers\n";
143  m_tml_global_comm.gather_packed(particleMMap);
144  console.Debug()
145  << "CLatticeMaster::visitParticlesOfType:"
146  << " gathered " << particleMMap.size() << " particles\n";
147 
148  console.Debug()
149  << "CLatticeMaster::visitParticlesOfType:"
150  << " visiting particle data\n";
151  for(
152  typename ParticleMMap::iterator iter=particleMMap.begin();
153  iter != particleMMap.end();
154  iter++
155  )
156  {
157  iter->second.visit(visitor);
158  }
159 
160  cmd.wait("visitParticles");
161  console.Debug() << "CLatticeMaster::visitParticlesOfType: exit\n";
162 }
163 
164 template <typename TmplVisitor>
165 void CLatticeMaster::visitParticles(
166  const IdVector &particleIdVector,
167  TmplVisitor &visitor
168 )
169 {
170  console.Debug() << "CLatticeMaster::visitParticles: enter\n";
171 
172  if (m_particle_type == "Basic")
173  {
174  visitParticlesOfType<TmplVisitor,CParticle>(particleIdVector, visitor);
175  }
176  else if (m_particle_type == "Rot")
177  {
178  visitParticlesOfType<TmplVisitor,CRotParticle>(particleIdVector, visitor);
179  }
180  else if (m_particle_type == "RotVi")
181  {
182  visitParticlesOfType<TmplVisitor,CRotParticleVi>(particleIdVector, visitor);
183  }
184  else if (m_particle_type == "RotTherm")
185  {
186  visitParticlesOfType<TmplVisitor,CRotThermParticle>(particleIdVector, visitor);
187  }
188  else
189  {
190  throw
191  std::runtime_error(
192  std::string("Unknown particle type: ") + m_particle_type
193  );
194  }
195  console.Debug() << "CLatticeMaster::visitParticles: exit\n";
196 }
197 
198 template <class TmplIterator, class TmplParticle>
199 void CLatticeMaster::addParticles(TmplIterator &particleIt)
200 {
201  CMPIBarrier barrier(m_global_comm);
202  CMPILCmdBuffer cmd_buffer(m_global_comm,m_global_rank);
203 
204  // cast storage pointer to correct type
205  vector<TmplParticle> particleVector;
206  console.XDebug()
207  << "CLatticeMaster::addParticles:"
208  << " Reserving vector memory for particles.\n";
209 
210  const int numBroadcastParticles = 50000;
211  particleVector.reserve(numBroadcastParticles);
212  console.XDebug()
213  << "CLatticeMaster::addParticles:"
214  << " Beginning add-particle loop..." << "\n";
215  for (int i = 0; particleIt.hasNext(); i++) {
216  const TmplParticle particle(particleIt.next());
217 
218  particleVector.push_back(particle);
219  if (((i+1) % 5000) == 0) {
220  console.XDebug()
221  << "CLatticeMaster::addParticles:"
222  << "Adding particle with id "
223  << particleVector.rbegin()->getID() << "\n";
224  }
225 
226  if (((i+1) % numBroadcastParticles) == 0)
227  {
228  console.XDebug() << "CLatticeMaster::addParticles:"
229  << " Broadcasting receive cmd...." << "\n";
230  cmd_buffer.broadcast(CMD_RECEIVEPARTICLES);
231  console.XDebug()
232  << "CLatticeMaster::addParticles: Broadcasting particles...." << "\n";
233  m_tml_global_comm.broadcast_cont_packed(particleVector);
234  barrier.wait("CLatticeMaster::addParticles: Post particle broadcast.");
235  barrier.wait("CLatticeMaster::addParticles: Post Command.");
236  particleVector.clear();
237  particleVector.reserve(numBroadcastParticles);
238  }
239  }
240  console.XDebug()
241  << "CLatticeMaster::addParticles: Done add-particle loop..." << "\n";
242  console.XDebug()
243  << "CLatticeMaster::addParticles: Broadcasting final particle-receive cmd"
244  << "\n";
245  cmd_buffer.broadcast(CMD_RECEIVEPARTICLES);
246  console.XDebug() << "CLatticeMaster::addParticles:"
247  << " Broadcasting final set of particles...\n";
248  m_tml_global_comm.broadcast_cont_packed(particleVector);
249  barrier.wait("Post final particle broadcast.");
250  barrier.wait("Post final particle-broadcast command.");
251  // -- build ntable
252  console.XDebug()
253  << "CLatticeMaster::addParticles: "
254  << "Building ntable (searchNeighbours)...\n";
255  searchNeighbors(true);
256  console.XDebug()
257  << "CLatticeMaster::addParticles: exit\n";
258 
259 }
260 
261 template <class TmplIterator>
262 void CLatticeMaster::addConnections(TmplIterator &connectionIt)
263 {
264  console.XDebug()
265  << "CLatticeMaster::addConnections: enter\n";
266 
267  CMPIBarrier barrier(m_global_comm);
268  CMPILCmdBuffer cmd_buffer(m_global_comm,m_global_rank);
269 
270  const int numBroadcastConnections = 100000;
271  vector<int> connectionBuffer;
272  connectionBuffer.reserve(numBroadcastConnections);
273  int i = 0;
274  for (; connectionIt.hasNext(); i++)
275  {
276  typename TmplIterator::value_type data = connectionIt.next();
277  connectionBuffer.push_back(data.getTag());
278  connectionBuffer.push_back(data.getP1Id());
279  connectionBuffer.push_back(data.getP2Id());
280  if ((i+1) % 50000 == 0)
281  {
282  console.XDebug() << "Adding connection number " << i << "\n";
283  }
284  if ((i+1) % numBroadcastConnections == 0)
285  {
286  console.XDebug() << "CLatticeMaster::addConnections:"
287  << " Broadcasting receive cmd...." << "\n";
288  cmd_buffer.broadcast(CMD_RECEIVECONNECTIONS);
289  console.XDebug()
290  << "CLatticeMaster::addConnections: Broadcasting connections...." << "\n";
291  m_tml_global_comm.broadcast_cont_packed(connectionBuffer);
292  barrier.wait("CLatticeMaster::addConnections: Post connection broadcast.");
293  barrier.wait("CLatticeMaster::addConnections: Post Command.");
294  connectionBuffer.clear();
295  connectionBuffer.reserve(numBroadcastConnections);
296  }
297  //m_temp_conn[data.getTag()].push_back(data.getP1Id());
298  //m_temp_conn[data.getTag()].push_back(data.getP2Id());
299  }
300  console.XDebug()
301  << "CLatticeMaster::addConnections: Done add-connection loop..." << "\n";
302  console.XDebug()
303  << "CLatticeMaster::addConnections: Broadcasting final connection-receive cmd"
304  << "\n";
305  cmd_buffer.broadcast(CMD_RECEIVECONNECTIONS);
306  console.XDebug() << "CLatticeMaster::addConnections:"
307  << " Broadcasting final set of connections...\n";
308  m_tml_global_comm.broadcast_cont_packed(connectionBuffer);
309  barrier.wait("Post final connection broadcast.");
310  barrier.wait("Post final connection-broadcast command.");
311 
312  console.XDebug()<< "Added " << i << " connections..." << "\n";
313  // -- build ntable
314  searchNeighbors(true);
315  console.XDebug()
316  << "CLatticeMaster::addConnections: exit\n";
317 }