Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Main/Inciter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Inciter, computational shock hydrodynamics tool, Charm++ main
9 : : chare.
10 : : \details Inciter, computational shock hydrodynamics tool, Charm++ main
11 : : chare. This file contains the definition of the Charm++ main chare,
12 : : equivalent to main() in Charm++-land.
13 : : */
14 : : // *****************************************************************************
15 : :
16 : : #include <unordered_map>
17 : : #include <vector>
18 : : #include <iostream>
19 : :
20 : : #include "Types.hpp"
21 : : #include "Init.hpp"
22 : : #include "QuinoaConfig.hpp"
23 : : #include "Timer.hpp"
24 : : #include "Exception.hpp"
25 : : #include "CGPDE.hpp"
26 : : #include "DGPDE.hpp"
27 : : #include "PDEStack.hpp"
28 : : #include "ProcessException.hpp"
29 : : #include "InciterPrint.hpp"
30 : : #include "InciterDriver.hpp"
31 : : #include "Inciter/CmdLine/Parser.hpp"
32 : : #include "Inciter/CmdLine/CmdLine.hpp"
33 : : #include "Inciter/InputDeck/InputDeck.hpp"
34 : : #include "ChareStateCollector.hpp"
35 : : #include "LBSwitch.hpp"
36 : :
37 : : #include "NoWarning/inciter.decl.h"
38 : :
39 : : #if defined(__clang__)
40 : : #pragma clang diagnostic push
41 : : #pragma clang diagnostic ignored "-Wmissing-variable-declarations"
42 : : #endif
43 : :
44 : : //! \brief Charm handle to the main proxy, facilitates call-back to finalize,
45 : : //! etc., must be in global scope, unique per executable
46 : : CProxy_Main mainProxy;
47 : :
48 : : //! Chare state collector Charm++ chare group proxy
49 : : tk::CProxy_ChareStateCollector stateProxy;
50 : :
51 : : //! Load balancer switch group proxy
52 : : tk::CProxy_LBSwitch LBSwitchProxy;
53 : :
54 : : //! If true, call and stack traces are to be output with exceptions
55 : : //! \note This is true by default so that the trace is always output between
56 : : //! program start and the Main ctor in which the user-input from command line
57 : : //! setting for this overrides this true setting.
58 : : bool g_trace = true;
59 : :
60 : : #if defined(__clang__)
61 : : #pragma clang diagnostic pop
62 : : #endif
63 : :
64 : : //! Inciter declarations and definitions
65 : : namespace inciter {
66 : :
67 : : //! Global-scope data. Initialized by the main chare and distibuted to all PEs
68 : : //! by the Charm++ runtime system. Though semantically not const, all these
69 : : //! global data should be considered read-only. See also
70 : : //! http://charm.cs.illinois.edu/manuals/html/charm++/manual.html. The data
71 : : //! below is global-scope because they must be available to all PEs which could
72 : : //! be on different machines.
73 : :
74 : : #if defined(__clang__)
75 : : #pragma clang diagnostic push
76 : : #pragma clang diagnostic ignored "-Wmissing-variable-declarations"
77 : : #endif
78 : :
79 : : //! Defaults of input deck, facilitates detection what is set by user
80 : : //! \details This object is in global scope, it contains the default of all
81 : : //! possible user input, and thus it is made available to all PEs for
82 : : //! convenience reasons. The runtime system distributes it to all PEs during
83 : : //! initialization. Once distributed, the object does not change.
84 : : ctr::InputDeck g_inputdeck_defaults;
85 : : //! Input deck filled by parser, containing all input data
86 : : //! \details This object is in global scope, it contains all of user input, and
87 : : //! thus it is made available to all PEs for convenience reasons. The runtime
88 : : //! system distributes it to all PEs during initialization. Once distributed,
89 : : //! the object does not change.
90 : : ctr::InputDeck g_inputdeck;
91 : : //! Partial differential equations using continuous Galerkin selected by user
92 : : //! \details This vector is in global scope, because it holds polymorphic
93 : : //! objects, and thus must be distributed to all PEs during initialization.
94 : : //! Once distributed by the runtime system, the objects do not change.
95 : : std::vector< CGPDE > g_cgpde;
96 : : //! Partial differential equations using discontinuous Galerkin selected by user
97 : : //! \details This vector is in global scope, because it holds polymorphic
98 : : //! objects, and thus must be distributed to all PEs during initialization.
99 : : //! Once distributed by the runtime system, the objects do not change.
100 : : std::vector< DGPDE > g_dgpde;
101 : :
102 : : #if defined(__clang__)
103 : : #pragma clang diagnostic pop
104 : : #endif
105 : :
106 : : //! \brief Pack/Unpack selected partial differential equations using continuous
107 : : //! Galerkin discretization.
108 : : //! \details This Pack/Unpack method (re-)creates the PDE factory since it needs
109 : : //! to (re-)bind function pointers on different processing elements. Therefore
110 : : //! we circumvent Charm's usual pack/unpack for this type, and thus sizing
111 : : //! does not make sense: sizing is a no-op. We could initialize the factory in
112 : : //! InciterDriver's constructor and let this function re-create the stack only
113 : : //! when unpacking, but that leads to repeating the same code twice: once in
114 : : //! InciterDriver's constructor, once here. Another option is to use this
115 : : //! pack/unpack routine to both initially create (when packing) and to
116 : : //! re-create (when unpacking) the factory, which eliminates the need for
117 : : //! pre-creating the object in InciterDriver's constructor and therefore
118 : : //! eliminates the repeated code. This explains the guard for sizing: the code
119 : : //! below is called for packing only (in serial) and packing and unpacking (in
120 : : //! parallel).
121 : : inline
122 : 899 : void operator|( PUP::er& p, std::vector< CGPDE >& eqs ) {
123 : : try {
124 [ + + ][ + - ]: 899 : if (!p.isSizing()) eqs = PDEStack().selectedCG();
[ + - ]
125 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
126 : 899 : }
127 : :
128 : : //! \brief Pack/Unpack selected partial differential equations using
129 : : //! discontinuous Galerkin discretization.
130 : : //! \details This Pack/Unpack method (re-)creates the PDE factory since it needs
131 : : //! to (re-)bind function pointers on different processing elements. Therefore
132 : : //! we circumvent Charm's usual pack/unpack for this type, and thus sizing
133 : : //! does not make sense: sizing is a no-op. We could initialize the factory in
134 : : //! InciterDriver's constructor and let this function re-create the stack only
135 : : //! when unpacking, but that leads to repeating the same code twice: once in
136 : : //! InciterDriver's constructor, once here. Another option is to use this
137 : : //! pack/unpack routine to both initially create (when packing) and to
138 : : //! re-create (when unpacking) the factory, which eliminates the need for
139 : : //! pre-creating the object in InciterDriver's constructor and therefore
140 : : //! eliminates the repeated code. This explains the guard for sizing: the code
141 : : //! below is called for packing only (in serial) and packing and unpacking (in
142 : : //! parallel).
143 : : inline
144 : 899 : void operator|( PUP::er& p, std::vector< DGPDE >& eqs ) {
145 : : try {
146 [ + + ][ + - ]: 899 : if (!p.isSizing()) eqs = PDEStack().selectedDG();
[ + - ]
147 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
148 : 899 : }
149 : :
150 : : } // inciter::
151 : :
152 : : //! \brief Charm++ main chare for the shock hydrodynamics executable, inciter.
153 : : //! \details In inciter the Charm++ runtime system is initialized only after the
154 : : //! mesh has been read in, partitioned, and the necessary data structures,
155 : : //! e.g., communication maps, have been generated. This delayed initialization
156 : : //! of the Charm++ runtime system is required since the mesh partitioning is
157 : : //! done by Zoltan, an MPI library. Note that this Charm++ main chare object
158 : : //! should not be in a namespace.
159 : : // cppcheck-suppress noConstructor
160 : : class Main : public CBase_Main {
161 : :
162 : : public:
163 : : //! \brief Constructor
164 : : //! \details Inciter's main chare constructor is the entry point of the
165 : : //! Charm++ portion of inciter, called by the Charm++ runtime system. The
166 : : //! constructor does basic initialization steps, prints out some useful
167 : : //! information to screen (in verbose mode), and instantiates a driver.
168 : : //! Since Charm++ is fully asynchronous, the constructor usually spawns
169 : : //! asynchronous objects and immediately exits. Thus in the body of the
170 : : //! main chare constructor we fire up an 'execute' chare, which then calls
171 : : //! back to Main::execute(). Finishing the main chare constructor the
172 : : //! Charm++ runtime system then starts the network-migration of all
173 : : //! global-scope data (if any). The execute chare calling back to
174 : : //! Main::execute() signals the end of the migration of the global-scope
175 : : //! data. Then we are ready to execute the driver. Since inciter is
176 : : //! parallel and asynchronous, its driver fires up additional Charm++
177 : : //! chare objects which then call back to Main::finalize() at some point
178 : : //! in the future when all work has been finished. finalize() then exits
179 : : //! by calling Charm++'s CkExit(), shutting down the runtime system.
180 : : //! \see http://charm.cs.illinois.edu/manuals/html/charm++/manual.html
181 : 233 : Main( CkArgMsg* msg )
182 : 233 : try :
183 : 466 : m_signal( tk::setSignalHandlers() ),
184 : : m_cmdline(),
185 : : // Parse command line into m_cmdline using default simple pretty printer
186 [ + - ]: 466 : m_cmdParser( msg->argc, msg->argv, tk::Print(), m_cmdline ),
187 : : // Create Inciter driver
188 : : m_driver( tk::Main< inciter::InciterDriver >
189 : : ( msg->argc, msg->argv,
190 : 233 : m_cmdline,
191 : : tk::HeaderType::INCITER,
192 [ + - ]: 466 : tk::inciter_executable(),
193 : : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io,
194 : 233 : tag::screen >(),
195 : : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io,
196 [ + - ]: 466 : tag::nrestart >() ) ),
197 : : // Start new timer measuring the total runtime
198 : : m_timer(1),
199 [ + - ][ + - ]: 699 : m_timestamp()
[ + - ][ + - ]
[ + - ]
200 : : {
201 [ + - ][ + - ]: 233 : delete msg;
202 : 233 : g_trace = m_cmdline.get< tag::trace >();
203 [ + - ]: 233 : tk::MainCtor( mainProxy, thisProxy, m_timer, m_cmdline,
204 [ + - ][ + - ]: 466 : CkCallback( CkIndex_Main::quiescence(), thisProxy ) );
205 : : // If quiescence detection is on or user requested it, create chare state
206 : : // collector Charm++ chare group
207 [ + + ][ + + ]: 233 : if ( m_cmdline.get< tag::chare >() || m_cmdline.get< tag::quiescence >() )
[ + + ]
208 [ + - ][ + - ]: 184 : stateProxy = tk::CProxy_ChareStateCollector::ckNew();
209 : : // Fire up an asynchronous execute object, which when created at some
210 : : // future point in time will call back to this->execute(). This is
211 : : // necessary so that this->execute() can access already migrated
212 : : // global-scope data.
213 [ + - ]: 233 : CProxy_execute::ckNew();
214 [ - - ]: 233 : } catch (...) { tk::processExceptionCharm(); }
215 : :
216 : : //! Migrate constructor: returning from a checkpoint
217 : 0 : explicit Main( CkMigrateMessage* msg ) : CBase_Main( msg ),
218 : 0 : m_signal( tk::setSignalHandlers() ),
219 : : m_cmdline(),
220 : : m_cmdParser( reinterpret_cast<CkArgMsg*>(msg)->argc,
221 : : reinterpret_cast<CkArgMsg*>(msg)->argv,
222 [ - - ]: 0 : tk::Print(),
223 : 0 : m_cmdline ),
224 : : m_driver( tk::Main< inciter::InciterDriver >
225 : : ( reinterpret_cast<CkArgMsg*>(msg)->argc,
226 : : reinterpret_cast<CkArgMsg*>(msg)->argv,
227 : 0 : m_cmdline,
228 : : tk::HeaderType::INCITER,
229 [ - - ]: 0 : tk::inciter_executable(),
230 : : inciter::g_inputdeck_defaults.get< tag::cmd,
231 : 0 : tag::io, tag::screen >(),
232 : : inciter::g_inputdeck.get< tag::cmd,
233 [ - - ]: 0 : tag::io, tag::nrestart >()+1 ) ),
234 : : m_timer(1),
235 [ - - ][ - - ]: 0 : m_timestamp()
[ - - ][ - - ]
236 : : {
237 : : // increase number of restarts (available for Transporter on PE 0)
238 : 0 : ++inciter::g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >();
239 : 0 : g_trace = m_cmdline.get< tag::trace >();
240 [ - - ]: 0 : tk::MainCtor( mainProxy, thisProxy, m_timer, m_cmdline,
241 [ - - ][ - - ]: 0 : CkCallback( CkIndex_Main::quiescence(), thisProxy ) );
242 : 0 : }
243 : :
244 : : //! Execute driver created and initialized by constructor
245 : 233 : void execute() {
246 : : try {
247 [ + - ][ + - ]: 233 : m_timestamp.emplace_back("Migrate global-scope data", m_timer[1].hms());
248 [ + - ]: 233 : m_driver.execute();
249 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
250 : 233 : }
251 : :
252 : : //! Towards normal exit but collect chare state first (if any)
253 : 233 : void finalize() {
254 [ + - ]: 233 : tk::finalize( m_cmdline, m_timer, stateProxy, m_timestamp,
255 : 233 : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >(),
256 : 233 : inciter::g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >(),
257 : 417 : CkCallback( CkIndex_Main::dumpstate(nullptr), thisProxy ) );
258 : 184 : }
259 : :
260 : : //! Entry method triggered when quiescence is detected
261 : 0 : void quiescence() {
262 : : try {
263 [ - - ]: 0 : stateProxy.collect( /* error= */ true,
264 [ - - ][ - - ]: 0 : CkCallback( CkIndex_Main::dumpstate(nullptr), thisProxy ) );
265 [ - - ]: 0 : } catch (...) { tk::processExceptionCharm(); }
266 : 0 : }
267 : :
268 : : //! Dump chare state
269 : 184 : void dumpstate( CkReductionMsg* msg ) {
270 : 184 : tk::dumpstate( m_cmdline,
271 : 184 : inciter::g_inputdeck_defaults.get< tag::cmd, tag::io, tag::screen >(),
272 : 184 : inciter::g_inputdeck.get< tag::cmd, tag::io, tag::nrestart >(),
273 : : msg );
274 : 0 : }
275 : :
276 : : /** @name Charm++ pack/unpack serializer member functions */
277 : : ///@{
278 : : //! \brief Pack/Unpack serialize member function
279 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
280 : : //! \note This is a Charm++ mainchare, pup() is thus only for
281 : : //! checkpoint/restart.
282 : 0 : void pup( PUP::er &p ) override {
283 : 0 : p | m_timer;
284 : 0 : }
285 : : //! \brief Pack/Unpack serialize operator|
286 : : //! \param[in,out] p Charm++'s PUP::er serializer object reference
287 : : //! \param[in,out] m Mainchare object reference
288 : : friend void operator|( PUP::er& p, Main& m ) { m.pup(p); }
289 : : //@}
290 : :
291 : : private:
292 : : int m_signal; //!< Used to set signal handlers
293 : : inciter::ctr::CmdLine m_cmdline; //!< Command line
294 : : inciter::CmdLineParser m_cmdParser; //!< Command line parser
295 : : inciter::InciterDriver m_driver; //!< Driver
296 : : std::vector< tk::Timer > m_timer; //!< Timers
297 : : //! Time stamps in h:m:s with labels
298 : : std::vector< std::pair< std::string, tk::Timer::Watch > > m_timestamp;
299 : : };
300 : :
301 : : //! \brief Charm++ chare execute
302 : : //! \details By the time this object is constructed, the Charm++ runtime system
303 : : //! has finished migrating all global-scoped read-only objects which happens
304 : : //! after the main chare constructor has finished.
305 : : class execute : public CBase_execute {
306 : : public:
307 : : //! Constructor
308 [ + - ]: 233 : execute() { mainProxy.execute(); }
309 : : //! Migrate constructor
310 : 0 : explicit execute( CkMigrateMessage* m ) : CBase_execute( m ) {}
311 : : };
312 : :
313 : : #include "NoWarning/inciter.def.h"
|
