Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Transporter.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Transporter drives the time integration of transport equations
9 : : \details Transporter drives the time integration of transport equations.
10 : : */
11 : : // *****************************************************************************
12 : : #ifndef Transporter_h
13 : : #define Transporter_h
14 : :
15 : : #include <map>
16 : : #include <vector>
17 : : #include <unordered_map>
18 : : #include <unordered_set>
19 : :
20 : : #include "Timer.hpp"
21 : : #include "Types.hpp"
22 : : #include "InciterPrint.hpp"
23 : : #include "Partitioner.hpp"
24 : : #include "Progress.hpp"
25 : : #include "Scheme.hpp"
26 : : #include "ContainerUtil.hpp"
27 : :
28 : : namespace inciter {
29 : :
30 : : //! Indices for progress report on mesh preparation
31 : : enum ProgMesh{ PART=0, DIST, REFINE, BND, COMM, MASK, REORD };
32 : : //! Prefixes for progress report on mesh preparation
33 : : static const std::array< std::string, 7 >
34 : : ProgMeshPrefix = {{ "p", "d", "r", "b", "c", "m", "r" }},
35 : : ProgMeshLegend = {{ "partition", "distribute", "refine", "bnd", "comm",
36 : : "mask", "reorder" }};
37 : :
38 : : //! Indices for progress report on workers preparation
39 : : enum ProgWork{ CREATE=0, BNDFACE, COMFAC, GHOST, ADJ };
40 : : //! Prefixes for progress report on workers preparation
41 : : static const std::array< std::string, 5 >
42 : : ProgWorkPrefix = {{ "c", "b", "f", "g", "a" }},
43 : : ProgWorkLegend = {{ "create", "bndface", "comfac", "ghost", "adj" }};
44 : :
45 : : //! Transporter drives the time integration of transport equations
46 : : class Transporter : public CBase_Transporter {
47 : :
48 : : public:
49 : : #if defined(__clang__)
50 : : #pragma clang diagnostic push
51 : : #pragma clang diagnostic ignored "-Wunused-parameter"
52 : : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
53 : : #elif defined(STRICT_GNUC)
54 : : #pragma GCC diagnostic push
55 : : #pragma GCC diagnostic ignored "-Wunused-parameter"
56 : : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
57 : : #elif defined(__INTEL_COMPILER)
58 : : #pragma warning( push )
59 : : #pragma warning( disable: 1478 )
60 : : #endif
61 : : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
62 : : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
63 : : Transporter_SDAG_CODE
64 : : #if defined(__clang__)
65 : : #pragma clang diagnostic pop
66 : : #elif defined(STRICT_GNUC)
67 : : #pragma GCC diagnostic pop
68 : : #elif defined(__INTEL_COMPILER)
69 : : #pragma warning( pop )
70 : : #endif
71 : :
72 : : //! Constructor
73 : : explicit Transporter();
74 : :
75 : : //! Migrate constructor: returning from a checkpoint
76 : : explicit Transporter( CkMigrateMessage* m );
77 : :
78 : : //! Reduction target: the mesh has been read from file on all PEs
79 : : void load( std::size_t meshid, std::size_t nelem );
80 : :
81 : : //! Reduction target: a mesh has been partitioned
82 : : void partitioned( std::size_t meshid );
83 : :
84 : : //! \brief Reduction target: all Solver (PEs) have computed the number of
85 : : //! chares they will recieve contributions from during linear solution
86 : : void partition();
87 : :
88 : : //! \brief Reduction target: all compute nodes have distrbuted their mesh
89 : : //! after partitioning
90 : : void distributed( std::size_t meshid );
91 : :
92 : : //! Reduction target: all Refiner chares have queried their boundary edges
93 : : void queriedRef( std::size_t meshid );
94 : : //! Reduction target: all Refiner chares have setup their boundary edges
95 : : void respondedRef( std::size_t meshid );
96 : :
97 : : //! Reduction target: all compute nodes have created the mesh refiners
98 : : void refinserted( std::size_t meshid, std::size_t error );
99 : :
100 : : //! Reduction target: all Discretization chares have been inserted
101 : : void discinserted( std::size_t meshid );
102 : :
103 : : //! Reduction target: all Discretization constructors have been called
104 : : void disccreated( std::size_t summeshid, std::size_t npoin );
105 : :
106 : : //! \brief Reduction target: all worker (derived discretization) chares have
107 : : //! been inserted
108 : : void workinserted( std::size_t meshid );
109 : :
110 : : //! \brief Reduction target: all Refiner chares have received a round
111 : : //! of edges, and have run their compatibility algorithm
112 : : void compatibility( std::size_t meshid, std::size_t modified );
113 : :
114 : : //! \brief Reduction target: all Refiner chares have matched/corrected
115 : : //! the tagging of chare-boundary edges, all chares are ready to perform
116 : : //! refinement.
117 : : void matched( std::size_t summeshid, std::size_t nextra, std::size_t nref,
118 : : std::size_t nderef, std::size_t sumrefmode );
119 : :
120 : : //! Compute surface integral across the whole problem and perform leak-test
121 : : void bndint( tk::real sx, tk::real sy, tk::real sz, tk::real cb,
122 : : tk::real summeshid );
123 : :
124 : : //! Reduction target: all chares have refined their mesh
125 : : void refined( std::size_t summeshid, std::size_t nelem, std::size_t npoin );
126 : :
127 : : //! \brief Reduction target: all worker chares have resized their own data
128 : : //! after AMR or ALE
129 : : void resized( std::size_t meshid );
130 : :
131 : : //! Reduction target: all worker chares have generated their own esup
132 : : void startEsup( std::size_t meshid );
133 : :
134 : : //! Reduction target: all Sorter chares have queried their boundary edges
135 : : void queried( std::size_t meshid );
136 : : //! Reduction target: all Sorter chares have setup their boundary edges
137 : : void responded( std::size_t meshid );
138 : :
139 : : //! Non-reduction target for receiving progress report on partitioning mesh
140 [ + - ]: 4 : void pepartitioned() { m_progMesh.inc< PART >( printer() ); }
141 : : //! Non-reduction target for receiving progress report on distributing mesh
142 [ + - ]: 4 : void pedistributed() { m_progMesh.inc< DIST >( printer() ); }
143 : : //! Non-reduction target for receiving progress report on finding bnd nodes
144 [ - - ]: 0 : void chbnd() { m_progMesh.inc< BND >( printer() ); }
145 : : //! Non-reduction target for receiving progress report on node ID comm map
146 [ + - ]: 4 : void chcomm() { m_progMesh.inc< COMM >( printer() ); }
147 : : //! Non-reduction target for receiving progress report on node ID mask
148 [ - - ]: 0 : void chmask() { m_progMesh.inc< MASK >( printer() ); }
149 : : //! Non-reduction target for receiving progress report on reordering mesh
150 [ - - ]: 0 : void chreordered() { m_progMesh.inc< REORD >( printer() ); }
151 : :
152 : : //! Non-reduction target for receiving progress report on creating workers
153 [ + - ]: 4 : void chcreated() { m_progWork.inc< CREATE >( printer() ); }
154 : : //! Non-reduction target for receiving progress report on finding bnd faces
155 [ - - ]: 0 : void chbndface() { m_progWork.inc< BNDFACE >( printer() ); }
156 : : //! Non-reduction target for receiving progress report on face communication
157 [ - - ]: 0 : void chcomfac() { m_progWork.inc< COMFAC >( printer() ); }
158 : : //! Non-reduction target for receiving progress report on sending ghost data
159 [ - - ]: 0 : void chghost() { m_progWork.inc< GHOST >( printer() ); }
160 : : //! Non-reduction target for receiving progress report on face adjacency
161 [ - - ]: 0 : void chadj() { m_progWork.inc< ADJ >( printer() ); }
162 : :
163 : : //! Reduction target indicating that the communication maps have been setup
164 : : void comfinal( std::size_t initial, std::size_t summeshid );
165 : :
166 : : //! Reduction target summing total mesh volume
167 : : void totalvol( tk::real v, tk::real initial, tk::real summeshid );
168 : :
169 : : //! \brief Reduction target yielding the minimum mesh statistics across
170 : : //! all workers
171 : : void minstat( tk::real d0, tk::real d1, tk::real d2, tk::real rmeshid );
172 : :
173 : : //! \brief Reduction target yielding the maximum mesh statistics across
174 : : //! all workers
175 : : void maxstat( tk::real d0, tk::real d1, tk::real d2, tk::real rmeshid );
176 : :
177 : : //! \brief Reduction target yielding the sum of mesh statistics across
178 : : //! all workers
179 : : void sumstat( tk::real d0, tk::real d1,
180 : : tk::real d2, tk::real d3,
181 : : tk::real d4, tk::real d5,
182 : : tk::real summeshid );
183 : :
184 : : //! \brief Reduction target yielding PDF of mesh statistics across all
185 : : //! workers
186 : : void pdfstat( CkReductionMsg* msg );
187 : :
188 : : //! Reduction target computing total volume of IC box
189 : : void boxvol( tk::real v, tk::real summeshid );
190 : :
191 : : //! \brief Reduction target optionally collecting diagnostics, e.g.,
192 : : //! residuals, from all worker chares
193 : : void diagnostics( CkReductionMsg* msg );
194 : :
195 : : //! Resume execution from checkpoint/restart files
196 : : void resume();
197 : :
198 : : //! Save checkpoint/restart files
199 : : void checkpoint( std::size_t finished, std::size_t meshid );
200 : :
201 : : //! Normal finish of time stepping
202 : : void finish( std::size_t meshid = 0 );
203 : :
204 : : /** @name Charm++ pack/unpack serializer member functions */
205 : : ///@{
206 : : //! \brief Pack/Unpack serialize member function
207 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
208 : : //! \note This is a Charm++ mainchare, pup() is thus only for
209 : : //! checkpoint/restart.
210 : 0 : void pup( PUP::er &p ) override {
211 : 0 : p | m_input;
212 : 0 : p | m_nchare;
213 : 0 : p | m_meshid;
214 : 0 : p | m_nload;
215 : 0 : p | m_npart;
216 : 0 : p | m_nstat;
217 : 0 : p | m_ndisc;
218 : 0 : p | m_nchk;
219 : 0 : p | m_ncom;
220 : 0 : p | m_ncit;
221 : 0 : p | m_nt0refit;
222 : 0 : p | m_ndtrefit;
223 : 0 : p | m_noutrefit;
224 : 0 : p | m_noutderefit;
225 : 0 : p | m_scheme;
226 : 0 : p | m_partitioner;
227 : 0 : p | m_refiner;
228 : 0 : p | m_meshwriter;
229 : 0 : p | m_sorter;
230 : 0 : p | m_nelem;
231 : 0 : p | m_npoin;
232 : : // returning from checkpoint
233 [ - - ][ - - ]: 0 : if (p.isUnpacking()) m_finished.resize( m_nchare.size(), 0 );
234 : 0 : p | m_meshvol;
235 : 0 : p | m_minstat;
236 : 0 : p | m_maxstat;
237 : 0 : p | m_avgstat;
238 : 0 : p | m_timer;
239 : 0 : }
240 : : //! \brief Pack/Unpack serialize operator|
241 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
242 : : //! \param[in,out] t Transporter object reference
243 : : friend void operator|( PUP::er& p, Transporter& t ) { t.pup(p); }
244 : : //@}
245 : :
246 : : private:
247 : : //! List of mesh files to be used for potentially multiple solvers
248 : : std::vector< std::string > m_input;
249 : : //! Number of worker chares (one per mesh)
250 : : std::vector< int > m_nchare;
251 : : //! Sum of mesh ids (across all chares, key) for each meshid (value)
252 : : std::unordered_map< std::size_t, std::size_t > m_meshid;
253 : : //! Number of mesh ref corr iter (one per mesh)
254 : : std::vector< std::size_t > m_ncit;
255 : : //! Number of meshes loaded
256 : : std::size_t m_nload;
257 : : //! Number of meshes partitioned
258 : : std::size_t m_npart;
259 : : //! Number of mesh statistics computed
260 : : std::size_t m_nstat;
261 : : //! Number of Discretization arrays created
262 : : std::size_t m_ndisc;
263 : : //! Number of worker arrays checkpointed
264 : : std::size_t m_nchk;
265 : : //! Number of worker arrays have finished setting up their comm maps
266 : : std::size_t m_ncom;
267 : : //! Number of t0ref mesh ref iters (one per mesh)
268 : : std::vector< std::size_t > m_nt0refit;
269 : : //! Number of dtref mesh ref iters (one per mesh)
270 : : std::vector< std::size_t > m_ndtrefit;
271 : : //! Number of outref mesh ref iters (one per mesh)
272 : : std::vector< std::size_t > m_noutrefit;
273 : : //! Number of outderef mesh ref iters (one per mesh)
274 : : std::vector< std::size_t > m_noutderefit;
275 : : //! Discretization scheme (one per mesh)
276 : : std::vector< Scheme > m_scheme;
277 : : //! Partitioner nodegroup proxies (one per mesh)
278 : : std::vector< CProxy_Partitioner > m_partitioner;
279 : : //! Mesh refiner array proxies (one per mesh)
280 : : std::vector< CProxy_Refiner > m_refiner;
281 : : //! Mesh writer nodegroup proxies (one per mesh)
282 : : std::vector< tk::CProxy_MeshWriter > m_meshwriter;
283 : : //! Mesh sorter array proxy (one per mesh)
284 : : std::vector< CProxy_Sorter > m_sorter;
285 : : //!< Number of mesh elements (per mesh)
286 : : std::vector< std::size_t > m_nelem;
287 : : //!< Number of mesh points (per mesh)
288 : : std::vector< std::size_t > m_npoin;
289 : : //!< Nonzero if finished with timestepping (one per mesh)
290 : : std::vector< std::size_t > m_finished;
291 : : //! Total mesh volume (one per mesh)
292 : : std::vector< tk::real > m_meshvol;
293 : : //! Minimum mesh statistics (one per mesh)
294 : : std::vector< std::array< tk::real, 3 > > m_minstat;
295 : : //! Maximum mesh statistics (one per mesh)
296 : : std::vector< std::array< tk::real, 3 > > m_maxstat;
297 : : //! Average mesh statistics (one per mesh)
298 : : std::vector< std::array< tk::real, 3 > > m_avgstat;
299 : : //! Timer tags
300 : : enum class TimerTag { MESH_READ=0 };
301 : : //! Timers
302 : : std::map< TimerTag, tk::Timer > m_timer;
303 : : //! Progress object for preparing mesh
304 : : tk::Progress< 7 > m_progMesh;
305 : : //! Progress object for preparing workers
306 : : tk::Progress< 5 > m_progWork;
307 : :
308 : : //! Create mesh partitioner and boundary condition object group
309 : : void createPartitioner();
310 : :
311 : : //! Configure and write diagnostics file header
312 : : void diagHeader();
313 : :
314 : : //! Echo configuration to screen
315 : : void info( const InciterPrint& print );
316 : :
317 : : //! Print out time integration header to screen
318 : : void inthead( const InciterPrint& print );
319 : :
320 : : //! Echo diagnostics on mesh statistics
321 : : void stat();
322 : :
323 : : //! Query variable names for all equation systems to be integrated
324 : : //! \param[in] eq Equation system whose variable names to query
325 : : //! \param[in,out] var Vector of strings to which we append the variable
326 : : //! names for this equation. We append as many strings as many scalar
327 : : //! variables are in the equation system given by eq.
328 : : template< class Eq >
329 : 233 : void varnames( const Eq& eq, std::vector< std::string >& var ) {
330 [ + - ]: 233 : auto o = eq.names();
331 [ + - ]: 233 : var.insert( end(var), begin(o), end(o) );
332 : 233 : }
333 : :
334 : : //! Create pretty printer specialized to Inciter
335 : : //! \return Pretty printer
336 : 1799 : InciterPrint printer() const {
337 : : const auto& def =
338 : 1799 : g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >();
339 : 1799 : auto nrestart = g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >();
340 : : return InciterPrint(
341 : 3598 : g_inputdeck.get< tag::cmd >().logname( def, nrestart ),
342 : 1799 : g_inputdeck.get< tag::cmd, tag::verbose >() ? std::cout : std::clog,
343 [ + - ][ + - ]: 5397 : std::ios_base::app );
344 : : }
345 : :
346 : : //! Function object for querying the side set ids the user configured
347 : : //! \details Used to query and collect the side set ids the user has
348 : : //! configured for all PDE types querying all BC types. Used on a
349 : : //! Carteisan product of 2 type lists: PDE types and BC types.
350 : : struct UserBC {
351 : : const ctr::InputDeck& inputdeck;
352 : : std::unordered_set< int >& userbc;
353 : 299 : explicit UserBC( const ctr::InputDeck& i, std::unordered_set< int >& u )
354 : 299 : : inputdeck(i), userbc(u) {}
355 : 5382 : template< typename U > void operator()( brigand::type_<U> ) {
356 : : using tag::param;
357 : : using eq = typename brigand::front< U >;
358 : : using bc = typename brigand::back< U >;
359 [ + + ]: 5647 : for (const auto& s : inputdeck.get< param, eq, tag::bc, bc >())
360 [ + + ][ + - ]: 982 : for (const auto& i : s) userbc.insert( std::stoi(i) );
[ + - ]
361 : 5382 : }
362 : : };
363 : :
364 : : //! Function object for querying the side set ids for time dependent BCs
365 : : //! \details Used to query and collect the side set ids the user has
366 : : //! configured for all PDE types querying time dependent BCs. Used on
367 : : //! PDE type list.
368 : : struct UserTimedepBC {
369 : : const ctr::InputDeck& inputdeck;
370 : : std::unordered_set< int >& userbc;
371 : 299 : explicit UserTimedepBC( const ctr::InputDeck& i,
372 : : std::unordered_set< int >& u )
373 : 299 : : inputdeck(i), userbc(u) {}
374 : 897 : template< typename eq > void operator()( brigand::type_<eq> ) {
375 : : using tag::param;
376 [ + + ]: 992 : for (const auto& sys : inputdeck.get< param, eq, tag::bctimedep >()) {
377 [ + + ]: 96 : for (const auto& b : sys) {
378 [ + + ][ + - ]: 2 : for (auto i : b.template get< tag::sideset >())
379 [ + - ][ + - ]: 1 : userbc.insert( std::stoi(i) );
380 : : }
381 : : }
382 : 897 : }
383 : : };
384 : :
385 : : //! Verify boundary condition (BC) side sets used exist in mesh file
386 : : bool matchBCs( std::map< int, std::vector< std::size_t > >& bnd );
387 : :
388 : : //! Print out mesh statistics
389 : : void meshstat( const std::string& header ) const;
390 : :
391 : : //! Generate list of input mesh filenames configured by the user
392 : : std::vector< std::string > input();
393 : :
394 : : //! Decide if ALE will need a linear solver
395 : : bool need_linearsolver() const;
396 : : };
397 : :
398 : : } // inciter::
399 : :
400 : : #endif // Transporter_h
|
