Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Main/Inciter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Inciter, computational shock hydrodynamics tool, Charm++ main
9 : : chare.
10 : : \details Inciter, computational shock hydrodynamics tool, Charm++ main
11 : : chare. This file contains the definition of the Charm++ main chare,
12 : : equivalent to main() in Charm++-land.
13 : : */
14 : : // *****************************************************************************
15 : :
16 : : #include <unordered_map>
17 : : #include <vector>
18 : : #include <iostream>
19 : :
20 : : #include "Types.hpp"
21 : : #include "Init.hpp"
22 : : #include "QuinoaConfig.hpp"
23 : : #include "Timer.hpp"
24 : : #include "Exception.hpp"
25 : : #include "CGPDE.hpp"
26 : : #include "DGPDE.hpp"
27 : : #include "FVPDE.hpp"
28 : : #include "PDEStack.hpp"
29 : : #include "ProcessException.hpp"
30 : : #include "InciterPrint.hpp"
31 : : #include "InciterDriver.hpp"
32 : : #include "Inciter/CmdLine/Parser.hpp"
33 : : #include "Inciter/CmdLine/CmdLine.hpp"
34 : : #include "Inciter/InputDeck/InputDeck.hpp"
35 : : #include "ChareStateCollector.hpp"
36 : : #include "LBSwitch.hpp"
37 : :
38 : : #include "NoWarning/inciter.decl.h"
39 : :
40 : : #if defined(__clang__)
41 : : #pragma clang diagnostic push
42 : : #pragma clang diagnostic ignored "-Wmissing-variable-declarations"
43 : : #endif
44 : :
45 : : //! \brief Charm handle to the main proxy, facilitates call-back to finalize,
46 : : //! etc., must be in global scope, unique per executable
47 : : CProxy_Main mainProxy;
48 : :
49 : : //! Chare state collector Charm++ chare group proxy
50 : : tk::CProxy_ChareStateCollector stateProxy;
51 : :
52 : : //! Load balancer switch group proxy
53 : : tk::CProxy_LBSwitch LBSwitchProxy;
54 : :
55 : : //! If true, call and stack traces are to be output with exceptions
56 : : //! \note This is true by default so that the trace is always output between
57 : : //! program start and the Main ctor in which the user-input from command line
58 : : //! setting for this overrides this true setting.
59 : : bool g_trace = true;
60 : :
61 : : #if defined(__clang__)
62 : : #pragma clang diagnostic pop
63 : : #endif
64 : :
65 : : //! Inciter declarations and definitions
66 : : namespace inciter {
67 : :
68 : : //! Global-scope data. Initialized by the main chare and distibuted to all PEs
69 : : //! by the Charm++ runtime system. Though semantically not const, all these
70 : : //! global data should be considered read-only. See also
71 : : //! http://charm.cs.illinois.edu/manuals/html/charm++/manual.html. The data
72 : : //! below is global-scope because they must be available to all PEs which could
73 : : //! be on different machines.
74 : :
75 : : #if defined(__clang__)
76 : : #pragma clang diagnostic push
77 : : #pragma clang diagnostic ignored "-Wmissing-variable-declarations"
78 : : #endif
79 : :
80 : : //! Defaults of input deck, facilitates detection what is set by user
81 : : //! \details This object is in global scope, it contains the default of all
82 : : //! possible user input, and thus it is made available to all PEs for
83 : : //! convenience reasons. The runtime system distributes it to all PEs during
84 : : //! initialization. Once distributed, the object does not change.
85 : : ctr::InputDeck g_inputdeck_defaults;
86 : : //! Lua Input deck filled by LuaParser, containing all input data
87 : : //! \details This object is in global scope, it contains all of user input, and
88 : : //! thus it is made available to all PEs for convenience reasons. The runtime
89 : : //! system distributes it to all PEs during initialization. Once distributed,
90 : : //! the object does not change.
91 : : ctr::InputDeck g_inputdeck;
92 : : //! Partial differential equations using continuous Galerkin selected by user
93 : : //! \details This vector is in global scope, because it holds polymorphic
94 : : //! objects, and thus must be distributed to all PEs during initialization.
95 : : //! Once distributed by the runtime system, the objects do not change.
96 : : std::vector< CGPDE > g_cgpde;
97 : : //! Partial differential equations using discontinuous Galerkin selected by user
98 : : //! \details This vector is in global scope, because it holds polymorphic
99 : : //! objects, and thus must be distributed to all PEs during initialization.
100 : : //! Once distributed by the runtime system, the objects do not change.
101 : : std::vector< DGPDE > g_dgpde;
102 : : //! Partial differential equations using finite volume selected by user
103 : : //! \details This vector is in global scope, because it holds polymorphic
104 : : //! objects, and thus must be distributed to all PEs during initialization.
105 : : //! Once distributed by the runtime system, the objects do not change.
106 : : std::vector< FVPDE > g_fvpde;
107 : :
108 : : #if defined(__clang__)
109 : : #pragma clang diagnostic pop
110 : : #endif
111 : :
112 : : //! \brief Pack/Unpack selected partial differential equations using continuous
113 : : //! Galerkin discretization.
114 : : //! \details This Pack/Unpack method (re-)creates the PDE factory since it needs
115 : : //! to (re-)bind function pointers on different processing elements. Therefore
116 : : //! we circumvent Charm's usual pack/unpack for this type, and thus sizing
117 : : //! does not make sense: sizing is a no-op. We could initialize the factory in
118 : : //! InciterDriver's constructor and let this function re-create the stack only
119 : : //! when unpacking, but that leads to repeating the same code twice: once in
120 : : //! InciterDriver's constructor, once here. Another option is to use this
121 : : //! pack/unpack routine to both initially create (when packing) and to
122 : : //! re-create (when unpacking) the factory, which eliminates the need for
123 : : //! pre-creating the object in InciterDriver's constructor and therefore
124 : : //! eliminates the repeated code. This explains the guard for sizing: the code
125 : : //! below is called for packing only (in serial) and packing and unpacking (in
126 : : //! parallel).
127 : : inline
128 : 883 : void operator|( PUP::er& p, std::vector< CGPDE >& eqs ) {
129 : : try {
130 [ + + ][ + - ]: 883 : if (!p.isSizing()) eqs = PDEStack().selectedCG();
[ + - ]
131 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
132 : 883 : }
133 : :
134 : : //! \brief Pack/Unpack selected partial differential equations using
135 : : //! discontinuous Galerkin discretization.
136 : : //! \details This Pack/Unpack method (re-)creates the PDE factory since it needs
137 : : //! to (re-)bind function pointers on different processing elements. Therefore
138 : : //! we circumvent Charm's usual pack/unpack for this type, and thus sizing
139 : : //! does not make sense: sizing is a no-op. We could initialize the factory in
140 : : //! InciterDriver's constructor and let this function re-create the stack only
141 : : //! when unpacking, but that leads to repeating the same code twice: once in
142 : : //! InciterDriver's constructor, once here. Another option is to use this
143 : : //! pack/unpack routine to both initially create (when packing) and to
144 : : //! re-create (when unpacking) the factory, which eliminates the need for
145 : : //! pre-creating the object in InciterDriver's constructor and therefore
146 : : //! eliminates the repeated code. This explains the guard for sizing: the code
147 : : //! below is called for packing only (in serial) and packing and unpacking (in
148 : : //! parallel).
149 : : inline
150 : 883 : void operator|( PUP::er& p, std::vector< DGPDE >& eqs ) {
151 : : try {
152 [ + + ][ + - ]: 883 : if (!p.isSizing()) eqs = PDEStack().selectedDG();
[ + - ]
153 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
154 : 883 : }
155 : :
156 : : //! \brief Pack/Unpack selected partial differential equations using
157 : : //! finite volume discretization.
158 : : //! \details This Pack/Unpack method (re-)creates the PDE factory since it needs
159 : : //! to (re-)bind function pointers on different processing elements. Therefore
160 : : //! we circumvent Charm's usual pack/unpack for this type, and thus sizing
161 : : //! does not make sense: sizing is a no-op. We could initialize the factory in
162 : : //! InciterDriver's constructor and let this function re-create the stack only
163 : : //! when unpacking, but that leads to repeating the same code twice: once in
164 : : //! InciterDriver's constructor, once here. Another option is to use this
165 : : //! pack/unpack routine to both initially create (when packing) and to
166 : : //! re-create (when unpacking) the factory, which eliminates the need for
167 : : //! pre-creating the object in InciterDriver's constructor and therefore
168 : : //! eliminates the repeated code. This explains the guard for sizing: the code
169 : : //! below is called for packing only (in serial) and packing and unpacking (in
170 : : //! parallel).
171 : : inline
172 : 883 : void operator|( PUP::er& p, std::vector< FVPDE >& eqs ) {
173 : : try {
174 [ + + ][ + - ]: 883 : if (!p.isSizing()) eqs = PDEStack().selectedFV();
[ + - ]
175 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
176 : 883 : }
177 : :
178 : : } // inciter::
179 : :
180 : : //! \brief Charm++ main chare for the shock hydrodynamics executable, inciter.
181 : : //! \details In inciter the Charm++ runtime system is initialized only after the
182 : : //! mesh has been read in, partitioned, and the necessary data structures,
183 : : //! e.g., communication maps, have been generated. This delayed initialization
184 : : //! of the Charm++ runtime system is required since the mesh partitioning is
185 : : //! done by Zoltan, an MPI library. Note that this Charm++ main chare object
186 : : //! should not be in a namespace.
187 : : // cppcheck-suppress noConstructor
188 : : class Main : public CBase_Main {
189 : :
190 : : public:
191 : : //! \brief Constructor
192 : : //! \details Inciter's main chare constructor is the entry point of the
193 : : //! Charm++ portion of inciter, called by the Charm++ runtime system. The
194 : : //! constructor does basic initialization steps, prints out some useful
195 : : //! information to screen (in verbose mode), and instantiates a driver.
196 : : //! Since Charm++ is fully asynchronous, the constructor usually spawns
197 : : //! asynchronous objects and immediately exits. Thus in the body of the
198 : : //! main chare constructor we fire up an 'execute' chare, which then calls
199 : : //! back to Main::execute(). Finishing the main chare constructor the
200 : : //! Charm++ runtime system then starts the network-migration of all
201 : : //! global-scope data (if any). The execute chare calling back to
202 : : //! Main::execute() signals the end of the migration of the global-scope
203 : : //! data. Then we are ready to execute the driver. Since inciter is
204 : : //! parallel and asynchronous, its driver fires up additional Charm++
205 : : //! chare objects which then call back to Main::finalize() at some point
206 : : //! in the future when all work has been finished. finalize() then exits
207 : : //! by calling Charm++'s CkExit(), shutting down the runtime system.
208 : : //! \see http://charm.cs.illinois.edu/manuals/html/charm++/manual.html
209 : 187 : Main( CkArgMsg* msg )
210 : 187 : try :
211 : 374 : m_signal( tk::setSignalHandlers() ),
212 : : m_cmdline(),
213 : : // Parse command line into m_cmdline using default simple pretty printer
214 [ + - ]: 374 : m_cmdParser( msg->argc, msg->argv, tk::Print(), m_cmdline ),
215 : : // Create Inciter driver
216 : : m_driver( tk::Main< inciter::InciterDriver >
217 : : ( msg->argc, msg->argv,
218 : 187 : m_cmdline,
219 : : tk::HeaderType::INCITER,
220 [ + - ]: 374 : tk::inciter_executable(),
221 : : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io,
222 : 187 : tag::screen >(),
223 : : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io,
224 [ + - ]: 374 : tag::nrestart >() ) ),
225 : : // Start new timer measuring the total runtime
226 : : m_timer(1),
227 [ + - ][ + - ]: 561 : m_timestamp()
[ + - ][ + - ]
[ + - ]
228 : : {
229 [ + - ][ + - ]: 187 : delete msg;
230 : 187 : g_trace = m_cmdline.get< tag::trace >();
231 [ + - ]: 187 : tk::MainCtor( mainProxy, thisProxy, m_timer, m_cmdline,
232 [ + - ][ + - ]: 374 : CkCallback( CkIndex_Main::quiescence(), thisProxy ) );
233 : : // If quiescence detection is on or user requested it, create chare state
234 : : // collector Charm++ chare group
235 [ + + ][ + + ]: 187 : if ( m_cmdline.get< tag::chare >() || m_cmdline.get< tag::quiescence >() )
[ + + ]
236 [ + - ][ + - ]: 140 : stateProxy = tk::CProxy_ChareStateCollector::ckNew();
237 : : // Fire up an asynchronous execute object, which when created at some
238 : : // future point in time will call back to this->execute(). This is
239 : : // necessary so that this->execute() can access already migrated
240 : : // global-scope data.
241 [ + - ]: 187 : CProxy_execute::ckNew();
242 [ - - ]: 187 : } catch (...) { tk::processExceptionCharm(); }
243 : :
244 : : //! Migrate constructor: returning from a checkpoint
245 : 2 : explicit Main( CkMigrateMessage* msg ) : CBase_Main( msg ),
246 : 4 : m_signal( tk::setSignalHandlers() ),
247 : : m_cmdline(),
248 : 2 : m_cmdParser( reinterpret_cast<CkArgMsg*>(msg)->argc,
249 : 2 : reinterpret_cast<CkArgMsg*>(msg)->argv,
250 [ + - ]: 4 : tk::Print(),
251 : 2 : m_cmdline ),
252 : : m_driver( tk::Main< inciter::InciterDriver >
253 : 2 : ( reinterpret_cast<CkArgMsg*>(msg)->argc,
254 : 2 : reinterpret_cast<CkArgMsg*>(msg)->argv,
255 : 2 : m_cmdline,
256 : : tk::HeaderType::INCITER,
257 [ + - ]: 4 : tk::inciter_executable(),
258 : : inciter::g_inputdeck_defaults.get< tag::cmd,
259 : 2 : tag::io, tag::screen >(),
260 : : inciter::g_inputdeck.get< tag::cmd,
261 [ + - ]: 4 : tag::io, tag::nrestart >()+1 ) ),
262 : : m_timer(1),
263 [ + - ][ + - ]: 6 : m_timestamp()
[ + - ][ + - ]
264 : : {
265 : : // increase number of restarts (available for Transporter on PE 0)
266 : 2 : ++inciter::g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >();
267 : 2 : g_trace = m_cmdline.get< tag::trace >();
268 [ + - ]: 2 : tk::MainCtor( mainProxy, thisProxy, m_timer, m_cmdline,
269 [ + - ][ + - ]: 4 : CkCallback( CkIndex_Main::quiescence(), thisProxy ) );
270 : 2 : }
271 : :
272 : : //! Execute driver created and initialized by constructor
273 : 187 : void execute() {
274 : : try {
275 [ + - ][ + - ]: 187 : m_timestamp.emplace_back("Migrate global-scope data", m_timer[1].hms());
276 [ + - ]: 187 : m_driver.execute();
277 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
278 : 187 : }
279 : :
280 : : //! Towards normal exit but collect chare state first (if any)
281 : 189 : void finalize() {
282 [ + - ]: 189 : tk::finalize( m_cmdline, m_timer, stateProxy, m_timestamp,
283 : 189 : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >(),
284 : 189 : inciter::g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >(),
285 : 331 : CkCallback( CkIndex_Main::dumpstate(nullptr), thisProxy ) );
286 : 142 : }
287 : :
288 : : //! Entry method triggered when quiescence is detected
289 : 0 : void quiescence() {
290 : : try {
291 [ - - ]: 0 : stateProxy.collect( /* error= */ true,
292 [ - - ][ - - ]: 0 : CkCallback( CkIndex_Main::dumpstate(nullptr), thisProxy ) );
293 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
294 : 0 : }
295 : :
296 : : //! Dump chare state
297 : 142 : void dumpstate( CkReductionMsg* msg ) {
298 : 142 : tk::dumpstate( m_cmdline,
299 : 142 : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >(),
300 : 142 : inciter::g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >(),
301 : : msg );
302 : 0 : }
303 : :
304 : : /** @name Charm++ pack/unpack serializer member functions */
305 : : ///@{
306 : : //! \brief Pack/Unpack serialize member function
307 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
308 : : //! \note This is a Charm++ mainchare, pup() is thus only for
309 : : //! checkpoint/restart.
310 : 119 : void pup( PUP::er &p ) override {
311 : 119 : p | m_timer;
312 : 119 : }
313 : : //! \brief Pack/Unpack serialize operator|
314 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
315 : : //! \param[in,out] m Mainchare object reference
316 : : friend void operator|( PUP::er& p, Main& m ) { m.pup(p); }
317 : : //@}
318 : :
319 : : private:
320 : : int m_signal; //!< Used to set signal handlers
321 : : inciter::ctr::CmdLine m_cmdline; //!< Command line
322 : : inciter::CmdLineParser m_cmdParser; //!< Command line parser
323 : : inciter::InciterDriver m_driver; //!< Driver
324 : : std::vector< tk::Timer > m_timer; //!< Timers
325 : : //! Time stamps in h:m:s with labels
326 : : std::vector< std::pair< std::string, tk::Timer::Watch > > m_timestamp;
327 : : };
328 : :
329 : : //! \brief Charm++ chare execute
330 : : //! \details By the time this object is constructed, the Charm++ runtime system
331 : : //! has finished migrating all global-scoped read-only objects which happens
332 : : //! after the main chare constructor has finished.
333 : : class execute : public CBase_execute {
334 : : public:
335 : : //! Constructor
336 [ + - ]: 187 : execute() { mainProxy.execute(); }
337 : : //! Migrate constructor
338 : 2 : explicit execute( CkMigrateMessage* m ) : CBase_execute( m ) {}
339 : : };
340 : :
341 : : #include "NoWarning/inciter.def.h"
|
