1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302 | // *****************************************************************************
/*!
\file src/Inciter/Ghosts.hpp
\copyright 2012-2015 J. Bakosi,
2016-2018 Los Alamos National Security, LLC.,
2019-2021 Triad National Security, LLC.
All rights reserved. See the LICENSE file for details.
\brief Declarations file for generating ghost data structures
\details Declarations file for asynchronous distributed
ghost data structures using Charm++.
There are a potentially large number of Ghosts Charm++ chares.
Each Ghosts chare gets a chunk of the full load, due to partiting the mesh.
The implementation uses the Charm++ runtime system and is fully
asynchronous, overlapping computation and communication. The algorithm
utilizes the structured dagger (SDAG) Charm++ functionality.
*/
// *****************************************************************************
#ifndef Ghosts_h
#define Ghosts_h
#include "Fields.hpp"
#include "FaceData.hpp"
#include "Discretization.hpp"
#include "NoWarning/ghosts.decl.h"
namespace inciter {
//! Ghosts Charm++ chare array used to determine ghost data structures
class Ghosts : public CBase_Ghosts {
public:
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-parameter"
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#elif defined(STRICT_GNUC)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#elif defined(__INTEL_COMPILER)
#pragma warning( push )
#pragma warning( disable: 1478 )
#endif
// Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
// charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
Ghosts_SDAG_CODE
#if defined(__clang__)
#pragma clang diagnostic pop
#elif defined(STRICT_GNUC)
#pragma GCC diagnostic pop
#elif defined(__INTEL_COMPILER)
#pragma warning( pop )
#endif
//! Constructor
explicit
Ghosts( const CProxy_Discretization& disc,
const std::map< int, std::vector< std::size_t > >& bface,
const std::vector< std::size_t >& triinpoel,
std::size_t nunk,
CkCallback cbDone );
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wundefined-func-template"
#endif
//! Migrate constructor
// cppcheck-suppress uninitMemberVar
explicit Ghosts( CkMigrateMessage* ) {}
#if defined(__clang__)
#pragma clang diagnostic pop
#endif
//! Local face & tet IDs associated to 3 global node IDs
//! \details This map stores tetrahedron cell faces (map key) and their
//! associated local face ID and inner local tet id adjacent to the face
//! (map value). A face is given by 3 global node IDs.
using FaceMap =
std::unordered_map< tk::UnsMesh::Face, // 3 global node IDs
std::array< std::size_t, 2 >, // local face & tet ID
tk::UnsMesh::Hash<3>,
tk::UnsMesh::Eq<3> >;
//! Storage type for refined mesh used for field output
struct OutMesh {
//! Element connectivity, local->global node ids, global->local nodes ids
tk::UnsMesh::Chunk chunk;
//! Node coordinates
tk::UnsMesh::Coords coord;
//! Triangle element connectivity
std::vector< std::size_t > triinpoel;
//! Boundary-face connectivity
std::map< int, std::vector< std::size_t > > bface;
//! Node communinaction map
tk::NodeCommMap nodeCommMap;
//! \brief Pack/Unpack serialize member function
//! \param[in,out] p Charm++'s PUP::er serializer object reference
void pup( PUP::er& p ) {<--- Parameter 'p' can be declared with const
p|chunk; p|coord; p|triinpoel; p|bface; p|nodeCommMap;
}
//! Destroyer
void destroy() {
tk::destroy( std::get<0>(chunk) );
tk::destroy( std::get<1>(chunk) );
tk::destroy( std::get<2>(chunk) );
tk::destroy( coord[0] );
tk::destroy( coord[1] );
tk::destroy( coord[2] );
tk::destroy( triinpoel );
tk::destroy( bface );
tk::destroy( nodeCommMap );
}
};
//! Discretization proxy
CProxy_Discretization m_disc;
//! Counter for number of unknowns on this chare (including ghosts)
std::size_t m_nunk;
//! Mesh connectivity extended
std::vector< std::size_t > m_inpoel;
//! Node coordinates extended
tk::UnsMesh::Coords m_coord;
//! Face data
FaceData m_fd;
//! Face geometry
tk::Fields m_geoFace;
//! Element geometry
tk::Fields m_geoElem;
//! Counter for number of faces on this chare (including chare boundaries)
std::size_t m_nfac;
//! Face & tet IDs associated to global node IDs of the face for each chare
//! \details This map stores not only the unique faces associated to
//! fellow chares, but also a newly assigned local face ID and adjacent
//! local tet ID.
std::unordered_map< int, FaceMap > m_bndFace;
//! Elements which are ghosts for other chares associated to those chare IDs
std::unordered_map< int, std::unordered_set< std::size_t > > m_sendGhost;
//! Local element id associated to ghost remote id charewise
//! \details This map associates the local element id (inner map value) to
//! the (remote) element id of the ghost (inner map key) based on the
//! chare id (outer map key) this remote element lies in.
std::unordered_map< int,
std::unordered_map< std::size_t, std::size_t > > m_ghost;
//! Expected ghost tet ids (used only in DEBUG)
std::set< std::size_t > m_exptGhost;
//! Map local ghost tet ids (value) and zero-based boundary ids (key)
std::unordered_map< std::size_t, std::size_t > m_bid;
//! Elements (value) surrounding point (key) data-structure
std::map< std::size_t, std::vector< std::size_t > > m_esup;
//! 1 if starting time stepping, 0 if during time stepping
std::size_t m_initial;
//1 Start setup of communication maps for cell-centered schemes
void startCommSetup();
//! Start sizing communication buffers and setting up ghost data
void resizeComm();
//! Receive unique set of faces we potentially share with/from another chare
void comfac( int fromch, const tk::UnsMesh::FaceSet& infaces );
//! Receive ghost data on chare boundaries from fellow chare
void comGhost( int fromch, const GhostData& ghost );
//! Receive requests for ghost data
void reqGhost();
//! Send all of our ghost data to fellow chares
void sendGhost();
//! Setup node-neighborhood (esup)
void nodeNeighSetup();
//! Receive element-surr-points data on chare boundaries from fellow chare
void comEsup( int fromch,
const std::unordered_map< std::size_t, std::vector< std::size_t > >&
bndEsup,
const std::unordered_map< std::size_t, std::vector< tk::real > >&
nodeBndCells );
/** @name Pack/unpack (Charm++ serialization) routines */
///@{
//! \brief Pack/Unpack serialize member function
//! \param[in,out] p Charm++'s PUP::er serializer object reference
void pup( PUP::er &p ) override {
p | m_disc;
p | m_nunk;
p | m_inpoel;
p | m_coord;
p | m_fd;
p | m_geoFace;
p | m_geoElem;
p | m_nfac;
p | m_bndFace;
p | m_sendGhost;
p | m_ghost;
p | m_exptGhost;
p | m_bid;
p | m_esup;
p | m_initial;
p | m_ncomfac;
p | m_nadj;
p | m_ncomEsup;
p | m_ipface;
p | m_ghostData;
p | m_ghostReq;
p | m_expChBndFace;
p | m_infaces;
p | m_esupc;
p | m_cbAfterDone;
}
//! \brief Pack/Unpack serialize operator|
//! \param[in,out] p Charm++'s PUP::er serializer object reference
//! \param[in,out] a Ghosts object reference
friend void operator|( PUP::er& p, Ghosts& a ) { a.pup(p); }
///@}
private:
using ncomp_t = tk::ncomp_t;
//! Counter for face adjacency communication map
std::size_t m_ncomfac;
//! Counter signaling that all ghost data have been received
std::size_t m_nadj;
//! Counter for element-surr-node adjacency communication map
std::size_t m_ncomEsup;
//! Internal + physical boundary faces (inverse of inpofa)
tk::UnsMesh::FaceSet m_ipface;
//! Ghost data associated to chare IDs we communicate with
std::unordered_map< int, GhostData > m_ghostData;
//! Number of chares requesting ghost data
std::size_t m_ghostReq;
//! Unique set of chare-boundary faces this chare is expected to receive
tk::UnsMesh::FaceSet m_expChBndFace;
//! Incoming communication buffer during chare-boundary face communication
std::unordered_map< int, tk::UnsMesh::FaceSet > m_infaces;
//! Communication buffer for esup data-structure
std::map< std::size_t, std::vector< std::size_t > > m_esupc;
//! Function call to continue with in Scheme when Ghosts is done
CkCallback m_cbAfterDone;
//! Access bound Discretization class pointer
Discretization* Disc() const {
Assert( m_disc[ thisIndex ].ckLocal() != nullptr, "ckLocal() null" );
return m_disc[ thisIndex ].ckLocal();
}
//! Compute chare-boundary faces
void bndFaces();
//! Setup own ghost data on this chare
void setupGhost();
//! Continue after face adjacency communication map completed on this chare
void faceAdj();
//! Compute partial boundary surface integral and sum across all chares
void bndIntegral();
//! Continue after node adjacency communication map completed on this chare
void adj();
//! Perform leak-test on chare boundary faces
bool leakyAdjacency();
//! Check if esuf of chare-boundary faces matches
bool faceMatch();
//! Verify that all chare-boundary faces have been received
bool receivedChBndFaces();
//! Find any chare for face (given by 3 global node IDs)
int findchare( const tk::UnsMesh::Face& t );
//! Check if entries in inpoel, inpofa and node-triplet are consistent
std::size_t nodetripletMatch(
const std::array< std::size_t, 2 >& id,
const tk::UnsMesh::Face& t );
//! Fill elements surrounding a face along chare boundary
void addEsuf(
const std::array< std::size_t, 2 >& id,
std::size_t ghostid );
//! Fill elements surrounding a element along chare boundary
void addEsuel(
const std::array< std::size_t, 2 >& id,
std::size_t ghostid,
const tk::UnsMesh::Face& t );
//! Fill face-geometry data along chare boundary
void addGeoFace(
const tk::UnsMesh::Face& t,
const std::array< std::size_t, 2 >& id );
};
} // inciter::
#endif // Ghosts_h
|