Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Transporter.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Transporter drives the time integration of transport equations
9 : : \details Transporter drives the time integration of transport equations.
10 : : */
11 : : // *****************************************************************************
12 : : #ifndef Transporter_h
13 : : #define Transporter_h
14 : :
15 : : #include <map>
16 : : #include <vector>
17 : : #include <unordered_map>
18 : : #include <unordered_set>
19 : :
20 : : #include "Timer.hpp"
21 : : #include "Types.hpp"
22 : : #include "InciterPrint.hpp"
23 : : #include "Partitioner.hpp"
24 : : #include "Progress.hpp"
25 : : #include "Scheme.hpp"
26 : : #include "ContainerUtil.hpp"
27 : : #include "Inciter/InputDeck/InputDeck.hpp"
28 : :
29 : : namespace inciter {
30 : :
31 : : extern ctr::InputDeck g_inputdeck;
32 : :
33 : : //! Indices for progress report on mesh preparation
34 : : enum ProgMesh{ PART=0, DIST, REFINE, BND, COMM, MASK, REORD };
35 : : //! Prefixes for progress report on mesh preparation
36 : : static const std::array< std::string, 7 >
37 : : ProgMeshPrefix = {{ "p", "d", "r", "b", "c", "m", "r" }},
38 : : ProgMeshLegend = {{ "partition", "distribute", "refine", "bnd", "comm",
39 : : "mask", "reorder" }};
40 : :
41 : : //! Indices for progress report on workers preparation
42 : : enum ProgWork{ CREATE=0, BNDFACE, COMFAC, GHOST, ADJ };
43 : : //! Prefixes for progress report on workers preparation
44 : : static const std::array< std::string, 5 >
45 : : ProgWorkPrefix = {{ "c", "b", "f", "g", "a" }},
46 : : ProgWorkLegend = {{ "create", "bndface", "comfac", "ghost", "adj" }};
47 : :
48 : : //! Transporter drives the time integration of transport equations
49 : : class Transporter : public CBase_Transporter {
50 : :
51 : : public:
52 : : #if defined(__clang__)
53 : : #pragma clang diagnostic push
54 : : #pragma clang diagnostic ignored "-Wunused-parameter"
55 : : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
56 : : #elif defined(STRICT_GNUC)
57 : : #pragma GCC diagnostic push
58 : : #pragma GCC diagnostic ignored "-Wunused-parameter"
59 : : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
60 : : #elif defined(__INTEL_COMPILER)
61 : : #pragma warning( push )
62 : : #pragma warning( disable: 1478 )
63 : : #endif
64 : : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
65 : : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
66 : : Transporter_SDAG_CODE
67 : : #if defined(__clang__)
68 : : #pragma clang diagnostic pop
69 : : #elif defined(STRICT_GNUC)
70 : : #pragma GCC diagnostic pop
71 : : #elif defined(__INTEL_COMPILER)
72 : : #pragma warning( pop )
73 : : #endif
74 : :
75 : : //! Constructor
76 : : explicit Transporter();
77 : :
78 : : //! Migrate constructor: returning from a checkpoint
79 : : explicit Transporter( CkMigrateMessage* m );
80 : :
81 : : //! Reduction target: the mesh has been read from file on all PEs
82 : : void load( std::size_t meshid, std::size_t nelem );
83 : :
84 : : //! Reduction target: a mesh has been partitioned
85 : : void partitioned( std::size_t meshid );
86 : :
87 : : //! \brief Reduction target: all Solver (PEs) have computed the number of
88 : : //! chares they will recieve contributions from during linear solution
89 : : void partition();
90 : :
91 : : //! \brief Reduction target: all compute nodes have distrbuted their mesh
92 : : //! after partitioning
93 : : void distributed( std::size_t meshid );
94 : :
95 : : //! Reduction target: all Refiner chares have queried their boundary edges
96 : : void queriedRef( std::size_t meshid );
97 : : //! Reduction target: all Refiner chares have setup their boundary edges
98 : : void respondedRef( std::size_t meshid );
99 : :
100 : : //! Reduction target: all compute nodes have created the mesh refiners
101 : : void refinserted( std::size_t meshid, std::size_t error );
102 : :
103 : : //! Reduction target: all Discretization chares have been inserted
104 : : void discinserted( std::size_t meshid );
105 : :
106 : : //! Reduction target: all Discretization constructors have been called
107 : : void disccreated( std::size_t summeshid, std::size_t npoin );
108 : :
109 : : //! \brief Reduction target: all worker (derived discretization) chares have
110 : : //! been inserted
111 : : void workinserted( std::size_t meshid );
112 : :
113 : : //! \brief Reduction target: all Refiner chares have received a round
114 : : //! of edges, and have run their compatibility algorithm
115 : : void compatibility( std::size_t meshid );
116 : :
117 : : //! \brief Reduction target: all Refiner chares have matched/corrected
118 : : //! the tagging of chare-boundary edges, all chares are ready to perform
119 : : //! refinement.
120 : : void matched( std::size_t summeshid, std::size_t nextra, std::size_t nref,
121 : : std::size_t nderef, std::size_t sumrefmode );
122 : :
123 : : //! Compute surface integral across the whole problem and perform leak-test
124 : : void bndint( tk::real sx, tk::real sy, tk::real sz, tk::real cb,
125 : : tk::real summeshid );
126 : :
127 : : //! Reduction target: all chares have refined their mesh
128 : : void refined( std::size_t summeshid, std::size_t nelem, std::size_t npoin );
129 : :
130 : : //! \brief Reduction target: all worker chares have resized their own data
131 : : //! after AMR or ALE
132 : : void resized( std::size_t meshid );
133 : :
134 : : //! Reduction target: all worker chares have generated their own esup
135 : : void startEsup( std::size_t meshid );
136 : :
137 : : //! Reduction target: all Sorter chares have queried their boundary edges
138 : : void queried( std::size_t meshid );
139 : : //! Reduction target: all Sorter chares have setup their boundary edges
140 : : void responded( std::size_t meshid );
141 : :
142 : : //! Non-reduction target for receiving progress report on partitioning mesh
143 [ - - ]: 0 : void pepartitioned() { m_progMesh.inc< PART >( printer() ); }
144 : : //! Non-reduction target for receiving progress report on distributing mesh
145 [ - - ]: 0 : void pedistributed() { m_progMesh.inc< DIST >( printer() ); }
146 : : //! Non-reduction target for receiving progress report on finding bnd nodes
147 [ - - ]: 0 : void chbnd() { m_progMesh.inc< BND >( printer() ); }
148 : : //! Non-reduction target for receiving progress report on node ID comm map
149 [ - - ]: 0 : void chcomm() { m_progMesh.inc< COMM >( printer() ); }
150 : : //! Non-reduction target for receiving progress report on node ID mask
151 [ - - ]: 0 : void chmask() { m_progMesh.inc< MASK >( printer() ); }
152 : : //! Non-reduction target for receiving progress report on reordering mesh
153 [ - - ]: 0 : void chreordered() { m_progMesh.inc< REORD >( printer() ); }
154 : :
155 : : //! Non-reduction target for receiving progress report on creating workers
156 [ - - ]: 0 : void chcreated() { m_progWork.inc< CREATE >( printer() ); }
157 : : //! Non-reduction target for receiving progress report on finding bnd faces
158 [ - - ]: 0 : void chbndface() { m_progWork.inc< BNDFACE >( printer() ); }
159 : : //! Non-reduction target for receiving progress report on face communication
160 [ - - ]: 0 : void chcomfac() { m_progWork.inc< COMFAC >( printer() ); }
161 : : //! Non-reduction target for receiving progress report on sending ghost data
162 [ - - ]: 0 : void chghost() { m_progWork.inc< GHOST >( printer() ); }
163 : : //! Non-reduction target for receiving progress report on face adjacency
164 [ - - ]: 0 : void chadj() { m_progWork.inc< ADJ >( printer() ); }
165 : :
166 : : // Reduction target indicating all "ghosts" insertions are done
167 : : void doneInsertingGhosts(std::size_t meshid);
168 : :
169 : : //! Reduction target indicating that the communication maps have been setup
170 : : void comfinal( std::size_t initial, std::size_t summeshid );
171 : :
172 : : //! Reduction target summing total mesh volume
173 : : void totalvol( tk::real v, tk::real initial, tk::real summeshid );
174 : :
175 : : //! \brief Reduction target yielding the minimum mesh statistics across
176 : : //! all workers
177 : : void minstat( tk::real d0, tk::real d1, tk::real d2, tk::real rmeshid );
178 : :
179 : : //! \brief Reduction target yielding the maximum mesh statistics across
180 : : //! all workers
181 : : void maxstat( tk::real d0, tk::real d1, tk::real d2, tk::real rmeshid );
182 : :
183 : : //! \brief Reduction target yielding the sum of mesh statistics across
184 : : //! all workers
185 : : void sumstat( tk::real d0, tk::real d1,
186 : : tk::real d2, tk::real d3,
187 : : tk::real d4, tk::real d5,
188 : : tk::real summeshid );
189 : :
190 : : //! \brief Reduction target yielding PDF of mesh statistics across all
191 : : //! workers
192 : : void pdfstat( CkReductionMsg* msg );
193 : :
194 : : //! Reduction target computing total volume of IC box
195 : : void boxvol( tk::real* meshdata, int n );
196 : :
197 : : //! Reduction target broadcasting to Schemes after mesh transfer
198 : : void solutionTransferred();
199 : :
200 : : //! Reduction target that computes minimum timestep across meshes
201 : : void minDtAcrossMeshes( tk::real* reducndata, int n );
202 : :
203 : : //! \brief Reduction target optionally collecting diagnostics, e.g.,
204 : : //! residuals, from all worker chares
205 : : void diagnostics( CkReductionMsg* msg );
206 : :
207 : : //! Resume execution from checkpoint/restart files
208 : : void resume();
209 : :
210 : : //! Save checkpoint/restart files
211 : : void checkpoint( std::size_t finished, std::size_t meshid );
212 : :
213 : : //! Normal finish of time stepping
214 : : void finish( std::size_t meshid = 0 );
215 : :
216 : : /** @name Charm++ pack/unpack serializer member functions */
217 : : ///@{
218 : : //! \brief Pack/Unpack serialize member function
219 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
220 : : //! \note This is a Charm++ mainchare, pup() is thus only for
221 : : //! checkpoint/restart.
222 : 119 : void pup( PUP::er &p ) override {
223 : 119 : p | m_input;
224 : 119 : p | m_nchare;
225 : 119 : p | m_meshid;
226 : 119 : p | m_nload;
227 : 119 : p | m_ntrans;
228 : 119 : p | m_ndtmsh;
229 : 119 : p | m_dtmsh;
230 : 119 : p | m_npart;
231 : 119 : p | m_nstat;
232 : 119 : p | m_ndisc;
233 : 119 : p | m_nchk;
234 : 119 : p | m_ncom;
235 : 119 : p | m_ncit;
236 : 119 : p | m_nt0refit;
237 : 119 : p | m_ndtrefit;
238 : 119 : p | m_noutrefit;
239 : 119 : p | m_noutderefit;
240 : 119 : p | m_scheme;
241 : 119 : p | m_partitioner;
242 : 119 : p | m_refiner;
243 : 119 : p | m_meshwriter;
244 : 119 : p | m_sorter;
245 : 119 : p | m_nelem;
246 : 119 : p | m_npoin;
247 : : // returning from checkpoint
248 [ + + ][ + - ]: 119 : if (p.isUnpacking()) m_finished.resize( m_nchare.size(), 0 );
249 : 119 : p | m_meshvol;
250 : 119 : p | m_minstat;
251 : 119 : p | m_maxstat;
252 : 119 : p | m_avgstat;
253 : 119 : p | m_timer;
254 : 119 : }
255 : : //! \brief Pack/Unpack serialize operator|
256 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
257 : : //! \param[in,out] t Transporter object reference
258 : : friend void operator|( PUP::er& p, Transporter& t ) { t.pup(p); }
259 : : //@}
260 : :
261 : : private:
262 : : //! List of mesh files to be used for potentially multiple solvers
263 : : std::vector< std::string > m_input;
264 : : //! Number of worker chares (one per mesh)
265 : : std::vector< int > m_nchare;
266 : : //! Sum of mesh ids (across all chares, key) for each meshid (value)
267 : : std::unordered_map< std::size_t, std::size_t > m_meshid;
268 : : //! Number of mesh ref corr iter (one per mesh)
269 : : std::vector< std::size_t > m_ncit;
270 : : //! Number of meshes loaded
271 : : std::size_t m_nload;
272 : : //! Number of meshes that have transferred solution
273 : : std::size_t m_ntrans;
274 : : //! Number of meshes that have computed their dt
275 : : std::size_t m_ndtmsh;
276 : : //! Minimum dt on each mesh (sized at each time step and then emptied)
277 : : std::vector< tk::real > m_dtmsh;
278 : : //! Number of meshes partitioned
279 : : std::size_t m_npart;
280 : : //! Number of mesh statistics computed
281 : : std::size_t m_nstat;
282 : : //! Number of Discretization arrays created
283 : : std::size_t m_ndisc;
284 : : //! Number of worker arrays checkpointed
285 : : std::size_t m_nchk;
286 : : //! Number of worker arrays have finished setting up their comm maps
287 : : std::size_t m_ncom;
288 : : //! Number of t0ref mesh ref iters (one per mesh)
289 : : std::vector< std::size_t > m_nt0refit;
290 : : //! Number of dtref mesh ref iters (one per mesh)
291 : : std::vector< std::size_t > m_ndtrefit;
292 : : //! Number of outref mesh ref iters (one per mesh)
293 : : std::vector< std::size_t > m_noutrefit;
294 : : //! Number of outderef mesh ref iters (one per mesh)
295 : : std::vector< std::size_t > m_noutderefit;
296 : : //! Discretization scheme (one per mesh)
297 : : std::vector< Scheme > m_scheme;
298 : : //! Partitioner nodegroup proxies (one per mesh)
299 : : std::vector< CProxy_Partitioner > m_partitioner;
300 : : //! Mesh refiner array proxies (one per mesh)
301 : : std::vector< CProxy_Refiner > m_refiner;
302 : : //! Mesh writer nodegroup proxies (one per mesh)
303 : : std::vector< tk::CProxy_MeshWriter > m_meshwriter;
304 : : //! Mesh sorter array proxy (one per mesh)
305 : : std::vector< CProxy_Sorter > m_sorter;
306 : : //!< Number of mesh elements (per mesh)
307 : : std::vector< std::size_t > m_nelem;
308 : : //!< Number of mesh points (per mesh)
309 : : std::vector< std::size_t > m_npoin;
310 : : //!< Nonzero if finished with timestepping (one per mesh)
311 : : std::vector< std::size_t > m_finished;
312 : : //! Total mesh volume (one per mesh)
313 : : std::vector< tk::real > m_meshvol;
314 : : //! Minimum mesh statistics (one per mesh)
315 : : std::vector< std::array< tk::real, 3 > > m_minstat;
316 : : //! Maximum mesh statistics (one per mesh)
317 : : std::vector< std::array< tk::real, 3 > > m_maxstat;
318 : : //! Average mesh statistics (one per mesh)
319 : : std::vector< std::array< tk::real, 3 > > m_avgstat;
320 : : //! Timer tags
321 : : enum class TimerTag { MESH_READ=0 };
322 : : //! Timers
323 : : std::map< TimerTag, tk::Timer > m_timer;
324 : : //! Progress object for preparing mesh
325 : : tk::Progress< 7 > m_progMesh;
326 : : //! Progress object for preparing workers
327 : : tk::Progress< 5 > m_progWork;
328 : :
329 : : //! Create mesh partitioner and boundary condition object group
330 : : void createPartitioner();
331 : :
332 : : //! Configure and write diagnostics file header
333 : : void diagHeader();
334 : :
335 : : //! Echo configuration to screen
336 : : void info( const InciterPrint& print );
337 : :
338 : : //! Print out time integration header to screen
339 : : void inthead( const InciterPrint& print );
340 : :
341 : : //! Echo diagnostics on mesh statistics
342 : : void stat();
343 : :
344 : : //! Query variable names for all equation systems to be integrated
345 : : //! \param[in] eq Equation system whose variable names to query
346 : : //! \param[in,out] var Vector of strings to which we append the variable
347 : : //! names for this equation. We append as many strings as many scalar
348 : : //! variables are in the equation system given by eq.
349 : : template< class Eq >
350 : 193 : void varnames( const Eq& eq, std::vector< std::string >& var ) {
351 [ + - ]: 193 : auto o = eq.names();
352 [ + - ]: 193 : var.insert( end(var), begin(o), end(o) );
353 : 193 : }
354 : :
355 : : //! Create pretty printer specialized to Inciter
356 : : //! \return Pretty printer
357 : 1424 : InciterPrint printer() const {
358 : : const auto& def =
359 : 1424 : g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >();
360 : 1424 : auto nrestart = g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >();
361 : : return InciterPrint(
362 : 2848 : g_inputdeck.get< tag::cmd >().logname( def, nrestart ),
363 : 1424 : g_inputdeck.get< tag::cmd, tag::verbose >() ? std::cout : std::clog,
364 [ + - ][ + - ]: 4272 : std::ios_base::app );
365 : : }
366 : :
367 : : //! Function object for querying the side set ids the user configured
368 : : //! \details Used to query and collect the side set ids the user has
369 : : //! configured for all PDE types querying all BC types. Used on a
370 : : //! Carteisan product of 2 type lists: PDE types and BC types.
371 : : struct UserBC {
372 : : const ctr::InputDeck& inputdeck;
373 : : std::unordered_set< int >& userbc;
374 : 275 : explicit UserBC( const ctr::InputDeck& i, std::unordered_set< int >& u )
375 : 275 : : inputdeck(i), userbc(u) {}
376 : 1925 : template< typename U > void operator()( brigand::type_<U> ) {
377 : 1925 : const auto& bcs = inputdeck.get< tag::bc >();
378 [ + + ]: 3906 : for (const auto& bci : bcs) {
379 [ + - ]: 1981 : userbc.insert( bci.get< U >().begin(), bci.get< U >().end() );
380 : : }
381 : 1925 : }
382 : : };
383 : :
384 : : //! Verify boundary condition (BC) side sets used exist in mesh file
385 : : bool matchBCs( std::map< int, std::vector< std::size_t > >& bnd );
386 : :
387 : : //! Print out mesh statistics
388 : : void meshstat( const std::string& header ) const;
389 : :
390 : : //! Generate list of input mesh filenames configured by the user
391 : : std::vector< std::string > input();
392 : :
393 : : //! Decide if ALE will need a linear solver
394 : : bool need_linearsolver() const;
395 : : };
396 : :
397 : : } // inciter::
398 : :
399 : : #endif // Transporter_h
|
