Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Transporter.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Transporter drives the time integration of transport equations
9 : : \details Transporter drives the time integration of transport equations.
10 : : */
11 : : // *****************************************************************************
12 : : #ifndef Transporter_h
13 : : #define Transporter_h
14 : :
15 : : #include <map>
16 : : #include <vector>
17 : : #include <unordered_map>
18 : : #include <unordered_set>
19 : :
20 : : #include "Timer.hpp"
21 : : #include "Types.hpp"
22 : : #include "InciterPrint.hpp"
23 : : #include "Partitioner.hpp"
24 : : #include "Progress.hpp"
25 : : #include "Scheme.hpp"
26 : : #include "ContainerUtil.hpp"
27 : : #include "Inciter/InputDeck/InputDeck.hpp"
28 : :
29 : : namespace inciter {
30 : :
31 : : extern ctr::InputDeck g_inputdeck;
32 : :
33 : : //! Indices for progress report on mesh preparation
34 : : enum ProgMesh{ PART=0, DIST, REFINE, BND, COMM, MASK, REORD };
35 : : //! Prefixes for progress report on mesh preparation
36 : : static const std::array< std::string, 7 >
37 : : ProgMeshPrefix = {{ "p", "d", "r", "b", "c", "m", "r" }},
38 : : ProgMeshLegend = {{ "partition", "distribute", "refine", "bnd", "comm",
39 : : "mask", "reorder" }};
40 : :
41 : : //! Indices for progress report on workers preparation
42 : : enum ProgWork{ CREATE=0, BNDFACE, COMFAC, GHOST, ADJ };
43 : : //! Prefixes for progress report on workers preparation
44 : : static const std::array< std::string, 5 >
45 : : ProgWorkPrefix = {{ "c", "b", "f", "g", "a" }},
46 : : ProgWorkLegend = {{ "create", "bndface", "comfac", "ghost", "adj" }};
47 : :
48 : : //! Transporter drives the time integration of transport equations
49 : : class Transporter : public CBase_Transporter {
50 : :
51 : : public:
52 : : #if defined(__clang__)
53 : : #pragma clang diagnostic push
54 : : #pragma clang diagnostic ignored "-Wunused-parameter"
55 : : #pragma clang diagnostic ignored "-Wdeprecated-declarations"
56 : : #elif defined(STRICT_GNUC)
57 : : #pragma GCC diagnostic push
58 : : #pragma GCC diagnostic ignored "-Wunused-parameter"
59 : : #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
60 : : #elif defined(__INTEL_COMPILER)
61 : : #pragma warning( push )
62 : : #pragma warning( disable: 1478 )
63 : : #endif
64 : : // Include Charm++ SDAG code. See http://charm.cs.illinois.edu/manuals/html/
65 : : // charm++/manual.html, Sec. "Structured Control Flow: Structured Dagger".
66 : : Transporter_SDAG_CODE
67 : : #if defined(__clang__)
68 : : #pragma clang diagnostic pop
69 : : #elif defined(STRICT_GNUC)
70 : : #pragma GCC diagnostic pop
71 : : #elif defined(__INTEL_COMPILER)
72 : : #pragma warning( pop )
73 : : #endif
74 : :
75 : : //! Constructor
76 : : explicit Transporter();
77 : :
78 : : //! Migrate constructor: returning from a checkpoint
79 : : explicit Transporter( CkMigrateMessage* m );
80 : :
81 : : //! Reduction target: the mesh has been read from file on all PEs
82 : : void load( std::size_t meshid, std::size_t nelem );
83 : :
84 : : //! Reduction target: a mesh has been partitioned
85 : : void partitioned( std::size_t meshid );
86 : :
87 : : //! \brief Reduction target: all Solver (PEs) have computed the number of
88 : : //! chares they will recieve contributions from during linear solution
89 : : void partition();
90 : :
91 : : //! \brief Reduction target: all compute nodes have distrbuted their mesh
92 : : //! after partitioning
93 : : void distributed( std::size_t meshid );
94 : :
95 : : //! Reduction target: all Refiner chares have queried their boundary edges
96 : : void queriedRef( std::size_t meshid );
97 : : //! Reduction target: all Refiner chares have setup their boundary edges
98 : : void respondedRef( std::size_t meshid );
99 : :
100 : : //! Reduction target: all compute nodes have created the mesh refiners
101 : : void refinserted( std::size_t meshid, std::size_t error );
102 : :
103 : : //! Reduction target: all Discretization chares have been inserted
104 : : void discinserted( std::size_t meshid );
105 : :
106 : : //! Reduction target: all Discretization constructors have been called
107 : : void disccreated( std::size_t summeshid, std::size_t npoin );
108 : :
109 : : //! \brief Reduction target: all worker (derived discretization) chares have
110 : : //! been inserted
111 : : void workinserted( std::size_t meshid );
112 : :
113 : : //! \brief Reduction target: all Refiner chares have received a round
114 : : //! of edges, and have run their compatibility algorithm
115 : : void compatibility( std::size_t meshid );
116 : :
117 : : //! \brief Reduction target: all Refiner chares have matched/corrected
118 : : //! the tagging of chare-boundary edges, all chares are ready to perform
119 : : //! refinement.
120 : : void matched( std::size_t summeshid, std::size_t nextra, std::size_t nref,
121 : : std::size_t nderef, std::size_t sumrefmode );
122 : :
123 : : //! Compute surface integral across the whole problem and perform leak-test
124 : : void bndint( tk::real sx, tk::real sy, tk::real sz, tk::real cb,
125 : : tk::real summeshid );
126 : :
127 : : //! Reduction target: all chares have refined their mesh
128 : : void refined( std::size_t summeshid, std::size_t nelem, std::size_t npoin );
129 : :
130 : : //! \brief Reduction target: all worker chares have resized their own data
131 : : //! after AMR or ALE
132 : : void resized( std::size_t meshid );
133 : :
134 : : //! Reduction target: all worker chares have generated their own esup
135 : : void startEsup( std::size_t meshid );
136 : :
137 : : //! Reduction target: all Sorter chares have queried their boundary edges
138 : : void queried( std::size_t meshid );
139 : : //! Reduction target: all Sorter chares have setup their boundary edges
140 : : void responded( std::size_t meshid );
141 : :
142 : : //! Non-reduction target for receiving progress report on partitioning mesh
143 [ - - ]: 0 : void pepartitioned() { m_progMesh.inc< PART >( printer() ); }
144 : : //! Non-reduction target for receiving progress report on distributing mesh
145 [ - - ]: 0 : void pedistributed() { m_progMesh.inc< DIST >( printer() ); }
146 : : //! Non-reduction target for receiving progress report on finding bnd nodes
147 [ - - ]: 0 : void chbnd() { m_progMesh.inc< BND >( printer() ); }
148 : : //! Non-reduction target for receiving progress report on node ID comm map
149 [ - - ]: 0 : void chcomm() { m_progMesh.inc< COMM >( printer() ); }
150 : : //! Non-reduction target for receiving progress report on node ID mask
151 [ - - ]: 0 : void chmask() { m_progMesh.inc< MASK >( printer() ); }
152 : : //! Non-reduction target for receiving progress report on reordering mesh
153 [ - - ]: 0 : void chreordered() { m_progMesh.inc< REORD >( printer() ); }
154 : :
155 : : //! Non-reduction target for receiving progress report on creating workers
156 [ - - ]: 0 : void chcreated() { m_progWork.inc< CREATE >( printer() ); }
157 : : //! Non-reduction target for receiving progress report on finding bnd faces
158 [ - - ]: 0 : void chbndface() { m_progWork.inc< BNDFACE >( printer() ); }
159 : : //! Non-reduction target for receiving progress report on face communication
160 [ - - ]: 0 : void chcomfac() { m_progWork.inc< COMFAC >( printer() ); }
161 : : //! Non-reduction target for receiving progress report on sending ghost data
162 [ - - ]: 0 : void chghost() { m_progWork.inc< GHOST >( printer() ); }
163 : : //! Non-reduction target for receiving progress report on face adjacency
164 [ - - ]: 0 : void chadj() { m_progWork.inc< ADJ >( printer() ); }
165 : :
166 : : // Reduction target indicating all "ghosts" insertions are done
167 : : void doneInsertingGhosts(std::size_t meshid);
168 : :
169 : : //! Reduction target indicating that the communication maps have been setup
170 : : void comfinal( std::size_t initial, std::size_t summeshid );
171 : :
172 : : //! Reduction target summing total mesh volume
173 : : void totalvol( tk::real v, tk::real initial, tk::real summeshid );
174 : :
175 : : //! \brief Reduction target yielding the minimum mesh statistics across
176 : : //! all workers
177 : : void minstat( tk::real d0, tk::real d1, tk::real d2, tk::real rmeshid );
178 : :
179 : : //! \brief Reduction target yielding the maximum mesh statistics across
180 : : //! all workers
181 : : void maxstat( tk::real d0, tk::real d1, tk::real d2, tk::real rmeshid );
182 : :
183 : : //! \brief Reduction target yielding the sum of mesh statistics across
184 : : //! all workers
185 : : void sumstat( tk::real d0, tk::real d1,
186 : : tk::real d2, tk::real d3,
187 : : tk::real d4, tk::real d5,
188 : : tk::real summeshid );
189 : :
190 : : //! \brief Reduction target yielding PDF of mesh statistics across all
191 : : //! workers
192 : : void pdfstat( CkReductionMsg* msg );
193 : :
194 : : //! Reduction target computing total volume of IC box
195 : : void boxvol( tk::real* meshdata, int n );
196 : :
197 : : //! Reduction target broadcasting to Schemes after mesh transfer
198 : : void solutionTransferred();
199 : :
200 : : // \brief Reduction target that computes minimum timestep across all meshes
201 : : // and sums up the forces on each mesh
202 : : void collectDtAndForces( CkReductionMsg* advMsg );
203 : :
204 : : //! \brief Reduction target optionally collecting diagnostics, e.g.,
205 : : //! residuals, from all worker chares
206 : : void diagnostics( CkReductionMsg* msg );
207 : :
208 : : //! Resume execution from checkpoint/restart files
209 : : void resume();
210 : :
211 : : //! Save checkpoint/restart files
212 : : void checkpoint( std::size_t finished, std::size_t meshid );
213 : :
214 : : //! Normal finish of time stepping
215 : : void finish( std::size_t meshid = 0 );
216 : :
217 : : /** @name Charm++ pack/unpack serializer member functions */
218 : : ///@{
219 : : //! \brief Pack/Unpack serialize member function
220 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
221 : : //! \note This is a Charm++ mainchare, pup() is thus only for
222 : : //! checkpoint/restart.
223 : 122 : void pup( PUP::er &p ) override {
224 : 122 : p | m_input;
225 : 122 : p | m_nchare;
226 : 122 : p | m_meshid;
227 : 122 : p | m_nload;
228 : 122 : p | m_ntrans;
229 : 122 : p | m_ndtmsh;
230 : 122 : p | m_dtmsh;
231 : 122 : p | m_npart;
232 : 122 : p | m_nstat;
233 : 122 : p | m_ndisc;
234 : 122 : p | m_nchk;
235 : 122 : p | m_ncom;
236 : 122 : p | m_ncit;
237 : 122 : p | m_nt0refit;
238 : 122 : p | m_ndtrefit;
239 : 122 : p | m_noutrefit;
240 : 122 : p | m_noutderefit;
241 : 122 : p | m_scheme;
242 : 122 : p | m_partitioner;
243 : 122 : p | m_refiner;
244 : 122 : p | m_meshwriter;
245 : 122 : p | m_sorter;
246 : 122 : p | m_nelem;
247 : 122 : p | m_npoin;
248 : : // returning from checkpoint
249 [ + + ]: 122 : if (p.isUnpacking()) m_finished.resize( m_nchare.size(), 0 );
250 : 122 : p | m_meshvol;
251 : 122 : p | m_minstat;
252 : 122 : p | m_maxstat;
253 : 122 : p | m_avgstat;
254 : 122 : p | m_timer;
255 : 122 : }
256 : : //! \brief Pack/Unpack serialize operator|
257 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
258 : : //! \param[in,out] t Transporter object reference
259 : : friend void operator|( PUP::er& p, Transporter& t ) { t.pup(p); }
260 : : //@}
261 : :
262 : : private:
263 : : //! List of mesh files to be used for potentially multiple solvers
264 : : std::vector< std::string > m_input;
265 : : //! Number of worker chares (one per mesh)
266 : : std::vector< int > m_nchare;
267 : : //! Sum of mesh ids (across all chares, key) for each meshid (value)
268 : : std::unordered_map< std::size_t, std::size_t > m_meshid;
269 : : //! Number of mesh ref corr iter (one per mesh)
270 : : std::vector< std::size_t > m_ncit;
271 : : //! Number of meshes loaded
272 : : std::size_t m_nload;
273 : : //! Number of meshes that have transferred solution
274 : : std::size_t m_ntrans;
275 : : //! Number of meshes that have computed their dt
276 : : std::size_t m_ndtmsh;
277 : : //! Minimum dt on each mesh (sized at each time step and then emptied)
278 : : std::vector< tk::real > m_dtmsh;
279 : : //! Number of meshes partitioned
280 : : std::size_t m_npart;
281 : : //! Number of mesh statistics computed
282 : : std::size_t m_nstat;
283 : : //! Number of Discretization arrays created
284 : : std::size_t m_ndisc;
285 : : //! Number of worker arrays checkpointed
286 : : std::size_t m_nchk;
287 : : //! Number of worker arrays have finished setting up their comm maps
288 : : std::size_t m_ncom;
289 : : //! Number of t0ref mesh ref iters (one per mesh)
290 : : std::vector< std::size_t > m_nt0refit;
291 : : //! Number of dtref mesh ref iters (one per mesh)
292 : : std::vector< std::size_t > m_ndtrefit;
293 : : //! Number of outref mesh ref iters (one per mesh)
294 : : std::vector< std::size_t > m_noutrefit;
295 : : //! Number of outderef mesh ref iters (one per mesh)
296 : : std::vector< std::size_t > m_noutderefit;
297 : : //! Discretization scheme (one per mesh)
298 : : std::vector< Scheme > m_scheme;
299 : : //! Partitioner nodegroup proxies (one per mesh)
300 : : std::vector< CProxy_Partitioner > m_partitioner;
301 : : //! Mesh refiner array proxies (one per mesh)
302 : : std::vector< CProxy_Refiner > m_refiner;
303 : : //! Mesh writer nodegroup proxies (one per mesh)
304 : : std::vector< tk::CProxy_MeshWriter > m_meshwriter;
305 : : //! Mesh sorter array proxy (one per mesh)
306 : : std::vector< CProxy_Sorter > m_sorter;
307 : : //!< Number of mesh elements (per mesh)
308 : : std::vector< std::size_t > m_nelem;
309 : : //!< Number of mesh points (per mesh)
310 : : std::vector< std::size_t > m_npoin;
311 : : //!< Nonzero if finished with timestepping (one per mesh)
312 : : std::vector< std::size_t > m_finished;
313 : : //! Total mesh volume (one per mesh)
314 : : std::vector< tk::real > m_meshvol;
315 : : //! Minimum mesh statistics (one per mesh)
316 : : std::vector< std::array< tk::real, 3 > > m_minstat;
317 : : //! Maximum mesh statistics (one per mesh)
318 : : std::vector< std::array< tk::real, 3 > > m_maxstat;
319 : : //! Average mesh statistics (one per mesh)
320 : : std::vector< std::array< tk::real, 3 > > m_avgstat;
321 : : //! Timer tags
322 : : enum class TimerTag { MESH_READ=0 };
323 : : //! Timers
324 : : std::map< TimerTag, tk::Timer > m_timer;
325 : : //! Progress object for preparing mesh
326 : : tk::Progress< 7 > m_progMesh;
327 : : //! Progress object for preparing workers
328 : : tk::Progress< 5 > m_progWork;
329 : :
330 : : //! Create mesh partitioner and boundary condition object group
331 : : void createPartitioner();
332 : :
333 : : //! Configure and write diagnostics file header
334 : : void diagHeader();
335 : :
336 : : //! Echo configuration to screen
337 : : void info( const InciterPrint& print );
338 : :
339 : : //! Print out time integration header to screen
340 : : void inthead( const InciterPrint& print );
341 : :
342 : : //! Echo diagnostics on mesh statistics
343 : : void stat();
344 : :
345 : : //! Query variable names for all equation systems to be integrated
346 : : //! \param[in] eq Equation system whose variable names to query
347 : : //! \param[in,out] var Vector of strings to which we append the variable
348 : : //! names for this equation. We append as many strings as many scalar
349 : : //! variables are in the equation system given by eq.
350 : : template< class Eq >
351 : : void varnames( const Eq& eq, std::vector< std::string >& var ) {
352 : : auto o = eq.names();
353 : : var.insert( end(var), begin(o), end(o) );
354 : : }
355 : :
356 : : //! Create pretty printer specialized to Inciter
357 : : //! \return Pretty printer
358 : 1442 : InciterPrint printer() const {
359 : : const auto& def =
360 : : g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >();
361 : 1442 : auto nrestart = g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >();
362 : : return InciterPrint(
363 : 1442 : g_inputdeck.get< tag::cmd >().logname( def, nrestart ),
364 : 1442 : g_inputdeck.get< tag::cmd, tag::verbose >() ? std::cout : std::clog,
365 [ - + ]: 2884 : std::ios_base::app );
366 : : }
367 : :
368 : : //! Function object for querying the side set ids the user configured
369 : : //! \details Used to query and collect the side set ids the user has
370 : : //! configured for all PDE types querying all BC types. Used on a
371 : : //! Carteisan product of 2 type lists: PDE types and BC types.
372 : : struct UserBC {
373 : : const ctr::InputDeck& inputdeck;
374 : : std::unordered_set< int >& userbc;
375 : : explicit UserBC( const ctr::InputDeck& i, std::unordered_set< int >& u )
376 [ + - ]: 274 : : inputdeck(i), userbc(u) {}
377 : 1644 : template< typename U > void operator()( brigand::type_<U> ) {
378 : 1644 : const auto& bcs = inputdeck.get< tag::bc >();
379 [ + + ]: 3312 : for (const auto& bci : bcs) {
380 : 1668 : userbc.insert( bci.get< U >().begin(), bci.get< U >().end() );
381 : : }
382 : 1644 : }
383 : : };
384 : :
385 : : //! Verify boundary condition (BC) side sets used exist in mesh file
386 : : bool matchBCs( std::map< int, std::vector< std::size_t > >& bnd );
387 : :
388 : : //! Print out mesh statistics
389 : : void meshstat( const std::string& header ) const;
390 : :
391 : : //! Generate list of input mesh filenames configured by the user
392 : : std::vector< std::string > input();
393 : :
394 : : //! Decide if ALE will need a linear solver
395 : : bool need_linearsolver() const;
396 : : };
397 : :
398 : : } // inciter::
399 : :
400 : : #endif // Transporter_h
|
