Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Sorter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Mesh sorter for global distributed mesh reordering
9 : : \see Sorter.h for more info.
10 : : */
11 : : // *****************************************************************************
12 : :
13 : : #include <vector>
14 : : #include <algorithm>
15 : :
16 : : #include "Sorter.hpp"
17 : : #include "Reorder.hpp"
18 : : #include "DerivedData.hpp"
19 : : #include "Inciter/InputDeck/InputDeck.hpp"
20 : :
21 : : namespace inciter {
22 : :
23 : : extern ctr::InputDeck g_inputdeck;
24 : :
25 : : } // inciter::
26 : :
27 : : using inciter::Sorter;
28 : :
29 : 2106 : Sorter::Sorter( std::size_t meshid,
30 : : const CProxy_Transporter& transporter,
31 : : const tk::CProxy_MeshWriter& meshwriter,
32 : : const tk::SorterCallback& cbs,
33 : : const std::vector< Scheme >& scheme,
34 : : CkCallback reorderRefiner,
35 : : const std::vector< std::size_t >& ginpoel,
36 : : const tk::UnsMesh::CoordMap& coordmap,
37 : : const tk::UnsMesh::Chunk& el,
38 : : const std::map< int, std::vector< std::size_t > >& bface,
39 : : const std::vector< std::size_t >& triinpoel,
40 : : const std::map< int, std::vector< std::size_t > >& bnode,
41 : 2106 : int nchare ) :
42 : : m_meshid( meshid ),
43 : : m_host( transporter ),
44 : : m_meshwriter( meshwriter ),
45 : : m_cbs( cbs ),
46 : : m_scheme( scheme ),
47 : : m_reorderRefiner( reorderRefiner ),
48 : : m_ginpoel( ginpoel ),
49 : : m_coordmap( coordmap ),
50 : : m_el( el ),
51 : : m_nbnd( 0 ),
52 : : m_bface( bface ),
53 : : m_triinpoel( triinpoel ),
54 : : m_bnode( bnode ),
55 : : m_nchare( nchare ),
56 : : m_nodeset( begin(ginpoel), end(ginpoel) ),
57 : : m_noffset( 0 ),
58 : : m_nodech(),
59 : : m_chnode(),
60 : : m_edgech(),
61 : : m_chedge(),
62 : : m_msum(),
63 : : m_reordcomm(),
64 : : m_start( 0 ),
65 : : m_newnodes(),
66 : : m_newcoordmap(),
67 : : m_reqnodes(),
68 : : m_lower( 0 ),
69 [ + - ][ + - ]: 2106 : m_upper( 0 )
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
70 : : // *****************************************************************************
71 : : // Constructor: prepare owned mesh node IDs for reordering
72 : : //! \param[in] meshid Mesh ID
73 : : //! \param[in] transporter Transporter (host) Charm++ proxy
74 : : //! \param[in] meshwriter Mesh writer Charm++ proxy
75 : : //! \param[in] cbs Charm++ callbacks for Sorter
76 : : //! \param[in] scheme Discretization schemes (one per mesh)
77 : : //! \param[in] reorderRefiner Callback to use to send reordered mesh to Refiner
78 : : //! \param[in] ginpoel Mesh connectivity (this chare) using global node IDs
79 : : //! \param[in] coordmap Mesh node coordinates (this chare) for global node IDs
80 : : //! \param[in] bface Face lists mapped to side set ids
81 : : //! \param[in] triinpoel Interconnectivity of points and boundary-faces
82 : : //! \param[in] bnode Node ids mapped to side set ids
83 : : //! \param[in] nchare Total number of Charm++ worker chares
84 : : // *****************************************************************************
85 : : {
86 : : // Ensure boundary face ids will not index out of face connectivity
87 [ + - ][ - + ]: 329690 : Assert( std::all_of( begin(m_bface), end(m_bface),
[ - - ][ - - ]
[ - - ]
88 : : [&](const auto& s)
89 : : { return std::all_of( begin(s.second), end(s.second),
90 : : [&](auto f){ return f*3+2 < m_triinpoel.size(); } ); } ),
91 : : "Boundary face data structures inconsistent" );
92 : 2106 : }
93 : :
94 : : void
95 : 2106 : Sorter::setup( std::size_t npoin )
96 : : // *****************************************************************************
97 : : // Setup chare mesh boundary node communication map
98 : : //! \param[in] npoin Total number of mesh points in mesh. Note that the number
99 : : //! of mesh points does not have to be exactly the total number of points in
100 : : //! the mesh. It can be a larger number, but not less. This is only used here
101 : : //! to assign nodes to workers that will assign ids to mesh nodes during node
102 : : //! reordering.
103 : : // *****************************************************************************
104 : : {
105 : : // Compute the number of nodes (chunksize) a chare will build a node
106 : : // communication map for. We compute two values of chunksize: one for when
107 : : // the global node ids are abounded between [0...npoin-1], inclusive, and
108 : : // another one for when the global node ids are assigned by a hash algorithm
109 : : // during initial mesh refinement. In the latter case, the maximum
110 : : // representable value of a std::size_t is assumed to be the large global node
111 : : // id and is used to compute the chunksize. To compute the bin id, we attempt
112 : : // to use the first chunksize first: if it gives a chare id that is
113 : : // (strictly) lower than the number of chares, that's good. If not, we compute
114 : : // the bin id based on the second chunksize, which almost always will give a
115 : : // bin id strictly lower than the number of chares, except if the global node
116 : : // id assigned by the hash algorithm in Refiner hits the maximum
117 : : // representable number in std::size_t. If that is the case, we just assign
118 : : // that node to the last chare.
119 : 2106 : auto N = static_cast< std::size_t >( m_nchare );
120 : : std::array< std::size_t, 2 > chunksize{{
121 : 2106 : npoin / N, std::numeric_limits< std::size_t >::max() / N }};
122 : :
123 : 2106 : const auto scheme = g_inputdeck.get< tag::discr, tag::scheme >();
124 : :
125 : : // Find chare-boundary nodes and edges of our mesh chunk. This algorithm
126 : : // collects the global mesh node ids and edges on the chare boundary. A node
127 : : // is on a chare boundary if it belongs to a face of a tetrahedron that has
128 : : // no neighbor tet at a face. The edge is on the chare boundary if its first
129 : : // edge-end point is on a chare boundary. The nodes are categorized to bins
130 : : // that will be sent to different chares to build point-to-point
131 : : // communication maps across all chares. The binning is determined by the
132 : : // global node id divided by the chunksizes. See discussion above on how we
133 : : // use two chunksizes for global node ids assigned by the hash algorithm in
134 : : // Refiner (if initial mesh refinement has been done).
135 : 4212 : tk::CommMaps chbnd;
136 [ + - ]: 4212 : auto el = tk::global2local( m_ginpoel ); // generate local mesh data
137 : 2106 : const auto& inpoel = std::get< 0 >( el ); // local connectivity
138 [ + - ]: 4212 : auto esup = tk::genEsup( inpoel, 4 ); // elements surrounding points
139 [ + - ]: 4212 : auto esuel = tk::genEsuelTet( inpoel, esup ); // elems surrounding elements
140 [ + + ]: 2452330 : for (std::size_t e=0; e<esuel.size()/4; ++e) {
141 : 2450224 : auto mark = e*4;
142 [ + + ]: 12251120 : for (std::size_t f=0; f<4; ++f)
143 [ + + ]: 9800896 : if (esuel[mark+f] == -1)
144 [ + + ]: 3985176 : for (std::size_t n=0; n<3; ++n) {
145 : 2988882 : auto g = m_ginpoel[ mark+tk::lpofa[f][n] ];
146 : 2988882 : auto bin = g / chunksize[0];
147 [ + + ]: 2988882 : if (bin >= N) bin = g / chunksize[1];
148 [ - + ]: 2988882 : if (bin >= N) bin = N - 1;
149 [ - + ][ - - ]: 2988882 : Assert( bin < N, "Will index out of number of chares" );
[ - - ][ - - ]
150 [ + - ]: 2988882 : auto& b = chbnd[ static_cast< int >( bin ) ];
151 [ + - ]: 2988882 : b.get< tag::node >().insert( g );
152 [ + + ]: 2988882 : if (scheme == ctr::SchemeType::ALECG) {
153 : 903090 : auto h = m_ginpoel[ mark + tk::lpofa[ f ][ tk::lpoet[n][1] ] ];
154 [ + - ]: 903090 : b.get< tag::edge >().insert( { std::min(g,h), std::max(g,h) } );
155 : : }
156 : : }
157 : : }
158 : :
159 : : // Send boundary data in bins to chares that will compute communication maps
160 : : // for the data in the bin. These bins form a distributed table. Note that
161 : : // we only send data to those chares that have data to work on. The receiving
162 : : // sides do not know in advance if they receive messages or not. Completion
163 : : // is detected by having the receiver respond back and counting the responses
164 : : // on the sender side, i.e., this chare.
165 : 2106 : m_nbnd = chbnd.size();
166 [ - + ]: 2106 : if (m_nbnd == 0)
167 [ - - ]: 0 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
168 : 0 : m_cbs.get< tag::queried >() );
169 : : else
170 [ + + ]: 28235 : for (const auto& [ targetchare, bnd ] : chbnd)
171 [ + - ][ + - ]: 26129 : thisProxy[ targetchare ].query( thisIndex, bnd );
172 : 2106 : }
173 : :
174 : : void
175 : 26129 : Sorter::query( int fromch, const tk::AllCommMaps& bnd )
176 : : // *****************************************************************************
177 : : // Incoming query for a list of mesh nodes for which this chare compiles node
178 : : // communication maps
179 : : //! \param[in] fromch Sender chare ID
180 : : //! \param[in] bnd Chare-boundary data from another chare
181 : : // *****************************************************************************
182 : : {
183 : : // Store incoming nodes in node->chare and its inverse, chare->node, maps
184 : 26129 : const auto& nodes = bnd.get< tag::node >();
185 [ + + ][ + - ]: 519675 : for (auto n : nodes) m_nodech[ n ].push_back( fromch );
[ + - ]
186 : 26129 : m_chnode[ fromch ].insert( begin(nodes), end(nodes) );
187 : :
188 : : // Store incoming edges in edge->chare and its inverse, chare->edge, maps
189 : 26129 : const auto& edges = bnd.get< tag::edge >();
190 [ + + ][ + - ]: 649376 : for (const auto& e : edges) m_edgech[ e ].push_back( fromch );
[ + - ]
191 : 26129 : m_chedge[ fromch ].insert( begin(edges), end(edges) );
192 : :
193 : : // Report back to chare message received from
194 [ + - ]: 26129 : thisProxy[ fromch ].recvquery();
195 : 26129 : }
196 : :
197 : : void
198 : 26129 : Sorter::recvquery()
199 : : // *****************************************************************************
200 : : // Receive receipt of boundary node lists to query
201 : : // *****************************************************************************
202 : : {
203 [ + + ]: 26129 : if (--m_nbnd == 0)
204 : 2106 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
205 : 2106 : m_cbs.get< tag::queried >() );
206 : 26129 : }
207 : :
208 : : void
209 : 2106 : Sorter::response()
210 : : // *****************************************************************************
211 : : // Respond to boundary node list queries
212 : : // *****************************************************************************
213 : : {
214 : 4212 : std::unordered_map< int, tk::CommMaps > exp;
215 : :
216 : : // Compute node communication map to be sent back to chares
217 [ + + ]: 28235 : for (const auto& [ neighborchare, bndnodes ] : m_chnode) {
218 [ + - ]: 26129 : auto& nc = exp[ neighborchare ];
219 [ + + ]: 519675 : for (auto n : bndnodes)
220 [ + - ][ + + ]: 1321546 : for (auto d : tk::cref_find(m_nodech,n))
221 [ + + ]: 828000 : if (d != neighborchare)
222 [ + - ][ + - ]: 334454 : nc[d].get< tag::node >().insert( n );
223 : : }
224 : :
225 : : // Compute edge communication map to be sent back to chares
226 [ + + ]: 28235 : for (const auto& [ neighborchare, bndedges ] : m_chedge) {
227 [ + - ]: 26129 : auto& ec = exp[ neighborchare ];
228 [ + + ]: 649376 : for (const auto& e : bndedges)
229 [ + - ][ + + ]: 1653946 : for (auto d : tk::cref_find(m_edgech,e))
230 [ + + ]: 1030699 : if (d != neighborchare)
231 [ + - ][ + - ]: 407452 : ec[d].get< tag::edge >().insert( e );
232 : : }
233 : :
234 : : // Send communication maps to chares that issued a query to us. Communication
235 : : // maps were computed above for those chares that queried this map from us.
236 : : // This data form a distributed table and we only work on a chunk of it. Note
237 : : // that we only send data back to those chares that have queried us. The
238 : : // receiving sides do not know in advance if the receive messages or not.
239 : : // Completion is detected by having the receiver respond back and counting
240 : : // the responses on the sender side, i.e., this chare.
241 : 2106 : m_nbnd = exp.size();
242 [ + + ]: 2106 : if (m_nbnd == 0)
243 [ + - ]: 700 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
244 : 700 : m_cbs.get< tag::responded >() );
245 : : else
246 [ + + ]: 27535 : for (const auto& [ targetchare, maps ] : exp)
247 [ + - ][ + - ]: 26129 : thisProxy[ targetchare ].bnd( thisIndex, maps );
248 : 2106 : }
249 : :
250 : : void
251 : 26129 : Sorter::bnd( int fromch, const tk::CommMaps& msum )
252 : : // *****************************************************************************
253 : : // Receive boundary node communication maps for our mesh chunk
254 : : //! \param[in] fromch Sender chare ID
255 : : //! \param[in] msum Communication map(s) assembled by chare fromch
256 : : // *****************************************************************************
257 : : {
258 [ + + ]: 130347 : for (const auto& [ neighborchare, maps ] : msum) {
259 [ + - ]: 104218 : auto& m = m_msum[ neighborchare ];
260 : 104218 : const auto& nodemap = maps.get< tag::node >();
261 [ + - ]: 104218 : m.get< tag::node >().insert( begin(nodemap), end(nodemap) );
262 : 104218 : const auto& edgemap = maps.get< tag::edge >();
263 [ + - ]: 104218 : m.get< tag::edge >().insert( begin(edgemap), end(edgemap) );
264 : : }
265 : :
266 : : // Report back to chare message received from
267 [ + - ]: 26129 : thisProxy[ fromch ].recvbnd();
268 : 26129 : }
269 : :
270 : : void
271 : 26129 : Sorter::recvbnd()
272 : : // *****************************************************************************
273 : : // Receive receipt of boundary node communication map
274 : : // *****************************************************************************
275 : : {
276 [ + + ]: 26129 : if (--m_nbnd == 0)
277 : 1406 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
278 : 1406 : m_cbs.get< tag::responded >() );
279 : 26129 : }
280 : :
281 : : void
282 : 2106 : Sorter::start()
283 : : // *****************************************************************************
284 : : // Start reordering (if enabled)
285 : : // *****************************************************************************
286 : : {
287 : : // Keep only those edges in edge comm map whose both end-points are in the
288 : : // node comm map
289 [ + + ]: 19804 : for (auto& [ neighborchare, maps ] : m_msum) {
290 : 17698 : const auto& nodes = maps.get< tag::node >();
291 : 35396 : tk::EdgeSet edges;
292 [ + + ]: 259556 : for (const auto& e : maps.get< tag::edge >())
293 [ + - ][ + - ]: 241858 : if (nodes.find(e[0]) != end(nodes) && nodes.find(e[1]) != end(nodes))
[ + - ][ + - ]
[ + - ]
294 [ + - ]: 241858 : edges.insert( e );
295 : 17698 : maps.get< tag::edge >() = std::move(edges);
296 : : }
297 : :
298 [ + + ]: 2106 : if (g_inputdeck.get< tag::cmd, tag::feedback >()) m_host.chcomm();
299 : :
300 : 2106 : tk::destroy( m_nodech );
301 : 2106 : tk::destroy( m_chnode );
302 : :
303 [ + + ]: 2106 : if (g_inputdeck.get< tag::discr, tag::pelocal_reorder >())
304 : 22 : mask(); // continue with mesh node reordering if requested (or required)
305 : : else
306 : 2084 : createDiscWorkers(); // skip mesh node reordering
307 : 2106 : }
308 : :
309 : : void
310 : 22 : Sorter::mask()
311 : : // *****************************************************************************
312 : : // Start preparing for mesh node reordering in parallel
313 : : // *****************************************************************************
314 : : {
315 : : // Compute asymmetric communcation map that will be used for reordering. This
316 : : // communication map is asymmetric because it associates global mesh node IDs
317 : : // to chares only with lower IDs than thisIndex. That is because this chare
318 : : // will need to receive new (reorderd) node IDs only from chares with lower
319 : : // IDs than thisIndex during node reordering. Since it only stores data for
320 : : // lower chare IDs, it is asymmetric. Note that because of this algorithm the
321 : : // type of m_msum is an ordered map, because of the std::none_of() algorithm
322 : : // needs to look at ALL chares this chare potentially communicates nodes with
323 : : // that have lower chare IDs that thisIndex. Since the map is ordered, it can
324 : : // walk through from the beginning of m_msum until the outer loop variable c,
325 : : // which is the chare ID the outer loop works on in a given cycle.
326 [ + + ]: 70 : for (auto c=m_msum.cbegin(); c!=m_msum.cend(); ++c)
327 [ + + ]: 48 : if (thisIndex > c->first) {
328 [ + - ]: 24 : auto& n = m_reordcomm[ c->first ];
329 [ + + ]: 310 : for (auto j : c->second.get< tag::node >())
330 [ + - ][ + + ]: 286 : if (std::none_of( m_msum.cbegin(), c,
331 : 194 : [j]( const auto& s ) {
332 : 194 : const auto& nodemap = s.second.template get< tag::node >();
333 [ + - ]: 194 : return nodemap.find(j) != end(nodemap); } ))
334 : : {
335 [ + - ]: 232 : n.insert(j);
336 : : }
337 [ - + ][ - - ]: 24 : if (n.empty()) m_reordcomm.erase( c->first );
338 : : }
339 : :
340 : : // Count up total number of nodes this chare will need to receive
341 : 22 : auto nrecv = tk::sumvalsize( m_reordcomm );
342 : :
343 [ - + ][ - - ]: 22 : if ( g_inputdeck.get< tag::cmd, tag::feedback >() ) m_host.chmask();
344 : :
345 : : // Compute number of mesh node IDs we will assign IDs to
346 : 22 : auto nuniq = m_nodeset.size() - nrecv;
347 : :
348 : : // Start computing offsets for node reordering
349 [ + - ]: 22 : thisProxy.offset( thisIndex, nuniq );
350 : 22 : }
351 : :
352 : : void
353 : 70 : Sorter::offset( int c, std::size_t u )
354 : : // *****************************************************************************
355 : : // Receive number of uniquely assigned global mesh node IDs from chares with
356 : : // lower IDs than thisIndex
357 : : //! \param[in] c Chare ID
358 : : //! \param[in] u Number of mesh node IDs chare c will assign IDs to
359 : : //! \details This function computes the offset each chare will need to start
360 : : //! assigning its new node IDs from. The offset for a chare is the
361 : : //! offset for the previous chare plus the number of node IDs the previous
362 : : //! chare (uniquely) assigns new IDs for minus the number of node IDs the
363 : : //! previous chare receives from others (lower chares). This is computed here
364 : : //! in a parallel/distributed fashion by each chare sending its number of node
365 : : //! IDs (that it uniquely assigns) to all chares. Note that each chare would
366 : : //! only need to send this information to chares with higher IDs, but instead
367 : : //! this function is called in a broadcast fashion, because that is more
368 : : //! efficient than individual calls to only chares with higher IDs. Therefore
369 : : //! when computing the offsets, we only count the lower chares. When this is
370 : : //! done, we have the precise asymmetric communication map as well as the
371 : : //! start offset on all chares and so we can start the distributed global mesh
372 : : //! node ID reordering.
373 : : // *****************************************************************************
374 : : {
375 [ + + ]: 70 : if (c < thisIndex) m_start += u;
376 [ + + ]: 70 : if (++m_noffset == m_nchare) reorder();
377 : 70 : }
378 : :
379 : : void
380 : 22 : Sorter::reorder()
381 : : // *****************************************************************************
382 : : // Reorder global mesh node IDs
383 : : // *****************************************************************************
384 : : {
385 : : // Activate SDAG waits for arriving requests from other chares requesting new
386 : : // node IDs for node IDs we assign new IDs to during reordering; and for
387 : : // computing/receiving lower and upper bounds of global node IDs our chare's
388 : : // linear system will operate on after reordering.
389 [ + - ][ + - ]: 22 : thisProxy[ thisIndex ].wait4prep();
390 : :
391 : : // Send out request for new global node IDs for nodes we do not reorder
392 [ + + ]: 46 : for (const auto& [ targetchare, nodes ] : m_reordcomm)
393 [ + - ][ + - ]: 24 : thisProxy[ targetchare ].request( thisIndex, nodes );
394 : :
395 : : // Lambda to decide if node is assigned a new ID by this chare. If node is not
396 : : // found in the asymmetric communication map, it is owned, i.e., this chare
397 : : // assigns its new id.
398 : 13420 : auto ownnode = [ this ]( std::size_t p ) {
399 : 6710 : return std::all_of( m_reordcomm.cbegin(), m_reordcomm.cend(),
400 : 1052 : [&](const auto& s)
401 [ + - ]: 7762 : { return s.second.find(p) == s.second.cend(); } );
402 : 22 : };
403 : :
404 : : // Reorder our chunk of the mesh node IDs. Looping through all of our node
405 : : // IDs, we test if we are to assign a new ID to a node ID, and if so, we
406 : : // assign a new ID, i.e., reorder, by constructing a map associating new to
407 : : // old IDs (m_newnodes). We also count up the reordered nodes, which serves as
408 : : // the new node id. We also store the node coordinates associated to the new
409 : : // node ID.
410 [ + + ]: 6732 : for (auto p : m_nodeset)
411 [ + - ][ + + ]: 6710 : if (ownnode(p)) {
412 [ + - ]: 6478 : m_newnodes[ p ] = m_start; // assign new node ID (reorder)
413 [ + - ][ + - ]: 6478 : m_newcoordmap.emplace( m_start, tk::cref_find(m_coordmap,p) );
414 : 6478 : ++m_start;
415 : : }
416 : :
417 : : // Trigger SDAG wait indicating that reordering our node IDs are complete
418 [ + - ]: 22 : reorderowned_complete();
419 : :
420 : : // If all our nodes have new IDs assigned, reordering complete on this chare
421 [ + + ][ + - ]: 22 : if (m_newnodes.size() == m_nodeset.size()) finish();
422 : 22 : }
423 : :
424 : : void
425 : 24 : Sorter::request( int c, const std::unordered_set< std::size_t >& nd )
426 : : // *****************************************************************************
427 : : // Request new global node IDs for old node IDs
428 : : //! \param[in] c Chare request coming from and to which we send new IDs to
429 : : //! \param[in] nd Set of old node IDs whose new IDs are requested
430 : : // *****************************************************************************
431 : : {
432 : : // Queue up requesting chare and node IDs
433 [ + - ]: 24 : m_reqnodes.push_back( { c, nd } );
434 : : // Trigger SDAG wait signaling that node IDs have been requested from us
435 : 24 : nodes_requested_complete();
436 : 24 : }
437 : :
438 : : void
439 : 24 : Sorter::prepare()
440 : : // *****************************************************************************
441 : : // Find new node IDs for old ones and return them to the requestor(s)
442 : : // *****************************************************************************
443 : : {
444 : : // Find and return new node IDs to sender
445 [ + + ]: 48 : for (const auto& [ requestorchare, nodes ] : m_reqnodes) {
446 : : std::unordered_map< std::size_t,
447 : 24 : std::tuple< std::size_t, tk::UnsMesh::Coord > > n;
448 [ + + ]: 256 : for (auto p : nodes) {
449 [ + - ]: 232 : auto newid = tk::cref_find( m_newnodes, p );
450 : : n.emplace( p,
451 [ + - ][ + - ]: 232 : std::make_tuple( newid, tk::cref_find(m_newcoordmap,newid) ) );
[ + - ]
452 : : }
453 [ + - ][ + - ]: 24 : thisProxy[ requestorchare ].neworder( n );
454 : : }
455 : :
456 : 24 : tk::destroy( m_reqnodes ); // Clear queue of requests just fulfilled
457 : :
458 : : // Re-enable SDAG wait for preparing new node requests
459 [ + - ]: 24 : thisProxy[ thisIndex ].wait4prep();
460 : :
461 : : // Re-enable trigger signaling that reordering of owned node IDs are
462 : : // complete right away
463 : 24 : reorderowned_complete();
464 : 24 : }
465 : :
466 : : void
467 : 24 : Sorter::neworder( const std::unordered_map< std::size_t,
468 : : std::tuple< std::size_t, tk::UnsMesh::Coord > >& nodes )
469 : : // *****************************************************************************
470 : : // Receive new (reordered) global node IDs
471 : : //! \param[in] nodes Map associating new to old node IDs
472 : : // *****************************************************************************
473 : : {
474 : : // Store new node IDs associated to old ones, and node coordinates associated
475 : : // to new node IDs.
476 [ + + ]: 256 : for (const auto& [ oldid, newnodes ] : nodes) {
477 : 232 : auto newid = std::get< 0 >( newnodes );
478 [ + - ]: 232 : m_newnodes[ oldid ] = newid;
479 [ + - ]: 232 : m_newcoordmap.emplace( newid, std::get< 1 >( newnodes ) );
480 : : }
481 : :
482 : : // If all our nodes have new IDs assigned, reorder complete on this PE
483 [ + + ]: 24 : if (m_newnodes.size() == m_nodeset.size()) finish();
484 : 24 : }
485 : :
486 : : void
487 : 22 : Sorter::finish()
488 : : // *****************************************************************************
489 : : // Compute final result of reordering
490 : : //! \details Reordering is now complete on this chare. We now remap all mesh
491 : : //! data to reflect the new ordering.
492 : : // *****************************************************************************
493 : : {
494 : : // Update elem connectivity with the reordered node IDs
495 : 22 : tk::remap( m_ginpoel, m_newnodes );
496 : :
497 : : // Update node coordinate map with the reordered IDs
498 : 22 : m_coordmap = m_newcoordmap;
499 : :
500 : : // Update mesh chunk data structure held in our state with new node order
501 : 22 : m_el = tk::global2local( m_ginpoel );
502 : :
503 : : // Update symmetric chare-node communication map with the reordered IDs
504 [ + + ]: 70 : for (auto& [ neighborchare, maps ] : m_msum) {
505 : :
506 : 96 : tk::NodeSet n;
507 [ + + ]: 620 : for (auto p : maps.get< tag::node >())
508 [ + - ][ + - ]: 572 : n.insert( tk::cref_find( m_newnodes, p ) );
509 : 48 : maps.get< tag::node >() = std::move( n );
510 : :
511 : 96 : tk::EdgeSet e;
512 [ - + ]: 48 : for (const auto& ed : maps.get< tag::edge >()) {
513 [ - - ]: 0 : e.insert( { tk::cref_find(m_newnodes,ed[0]),
514 [ - - ][ - - ]: 0 : tk::cref_find(m_newnodes,ed[1]) } );
515 : : }
516 : 48 : maps.get< tag::edge >() = std::move( e );
517 : :
518 : : }
519 : :
520 : : // Update boundary face-node connectivity with the reordered node IDs
521 : 22 : tk::remap( m_triinpoel, m_newnodes );
522 : :
523 : : // Update boundary node lists with the reordered node IDs
524 [ + + ][ + - ]: 54 : for (auto& [ setid, nodes ] : m_bnode) tk::remap( nodes, m_newnodes );
525 : :
526 : : // Update mesh in Refiner after reordering
527 : 22 : m_reorderRefiner.send();
528 : :
529 : : // Progress report to host
530 [ - + ]: 22 : if ( g_inputdeck.get< tag::cmd, tag::feedback >() ) m_host.chreordered();
531 : :
532 : 22 : createDiscWorkers();
533 : 22 : }
534 : :
535 : : void
536 : 22 : Sorter::mesh( std::vector< std::size_t >& ginpoel,
537 : : tk::UnsMesh::CoordMap& coordmap,
538 : : std::vector< std::size_t >& triinpoel,
539 : : std::map< int, std::vector< std::size_t > >& bnode )
540 : : // *****************************************************************************
541 : : // Update mesh data we hold for whoever calls this function
542 : : //! \param[in,out] ginpoel Mesh connectivity using global IDs
543 : : //! \param[in,out] coordmap Map of mesh node coordinates
544 : : //! \param[in,out] triinpoel Boundary face-node connectivity
545 : : //! \param[in] bnode Node lists of side sets
546 : : // *****************************************************************************
547 : : {
548 : 22 : ginpoel = m_ginpoel;
549 : 22 : coordmap = m_coordmap;
550 : 22 : triinpoel = m_triinpoel;
551 : 22 : bnode = m_bnode;
552 : 22 : }
553 : :
554 : : void
555 : 2106 : Sorter::createDiscWorkers()
556 : : // *****************************************************************************
557 : : // Create Discretization chare array elements on this PE
558 : : //! \details We create chare array elements by calling the insert() member
559 : : //! function, which allows specifying the PE on which the array element is
560 : : //! created. and we send each chare array element the chunk of mesh it will
561 : : //! operate on.
562 : : // *****************************************************************************
563 : : {
564 : 4212 : std::vector< CProxy_Discretization > disc;
565 [ + + ][ + - ]: 4212 : for (auto& d : m_scheme) disc.push_back( d.disc() );
566 : :
567 : : // Create worker array element using Charm++ dynamic chare array element
568 : : // insertion: last arg: PE chare is created on. See also Charm++ manual, Sec.
569 : : // "Dynamic Insertion".
570 : :
571 [ + - ][ + - ]: 4212 : m_scheme[m_meshid].disc()[ thisIndex ].insert( m_meshid, disc,
572 : 2106 : m_scheme[m_meshid].fct(), m_scheme[m_meshid].ale(),
573 : 2106 : m_scheme[m_meshid].conjugategradients(), m_host,
574 : 2106 : m_meshwriter, m_coordmap, m_el, m_msum, m_nchare );
575 : :
576 [ + - ]: 2106 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
577 : 2106 : m_cbs.get< tag::discinserted >() );
578 : 2106 : }
579 : :
580 : : void
581 : 2106 : Sorter::createWorkers()
582 : : // *****************************************************************************
583 : : // Create worker chare array element
584 : : // *****************************************************************************
585 : : {
586 : : // Make sure (bound) base is already created and accessible
587 [ + - ][ - + ]: 2106 : Assert( m_scheme[m_meshid].disc()[thisIndex].ckLocal() != nullptr,
[ - - ][ - - ]
[ - - ]
588 : : "About to pass nullptr" );
589 : :
590 : : // Create worker array element using Charm++ dynamic chare array element
591 : : // insertion: 1st arg: chare id, other args: Discretization's child ctor args.
592 : : // See also Charm++ manual, Sec. "Dynamic Insertion".
593 : :
594 [ + - ]: 4212 : m_scheme[m_meshid].insert( thisIndex, m_scheme[m_meshid].disc(), m_bface,
595 : 2106 : m_bnode, m_triinpoel );
596 : :
597 [ + + ]: 2106 : if ( g_inputdeck.get< tag::cmd, tag::feedback >() ) m_host.chcreated();
598 : :
599 : 2106 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
600 : 2106 : m_cbs.get< tag::workinserted >() );
601 : :
602 : : // Free up some memory
603 : 2106 : tk::destroy( m_ginpoel );
604 : 2106 : tk::destroy( m_coordmap );
605 : 2106 : tk::destroy( m_bface );
606 : 2106 : tk::destroy( m_triinpoel );
607 : 2106 : tk::destroy( m_bnode );
608 : 2106 : tk::destroy( m_nodeset );
609 : 2106 : tk::destroy( m_nodech );
610 : 2106 : tk::destroy( m_chnode );
611 : 2106 : tk::destroy( m_msum );
612 : 2106 : tk::destroy( m_reordcomm );
613 : 2106 : tk::destroy( m_newnodes );
614 : 2106 : tk::destroy( m_reqnodes );
615 : 2106 : }
616 : :
617 : : #include "NoWarning/sorter.def.h"
|
