Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Inciter/Sorter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Mesh sorter for global distributed mesh reordering
9 : : \see Sorter.h for more info.
10 : : */
11 : : // *****************************************************************************
12 : :
13 : : #include <vector>
14 : : #include <algorithm>
15 : :
16 : : #include "Sorter.hpp"
17 : : #include "Reorder.hpp"
18 : : #include "DerivedData.hpp"
19 : : #include "Inciter/InputDeck/InputDeck.hpp"
20 : :
21 : : namespace inciter {
22 : :
23 : : extern ctr::InputDeck g_inputdeck;
24 : :
25 : : } // inciter::
26 : :
27 : : using inciter::Sorter;
28 : :
29 : 1943 : Sorter::Sorter( std::size_t meshid,
30 : : const CProxy_Transporter& transporter,
31 : : const tk::CProxy_MeshWriter& meshwriter,
32 : : const tk::SorterCallback& cbs,
33 : : const std::vector< Scheme >& scheme,
34 : : CkCallback reorderRefiner,
35 : : const std::vector< std::size_t >& ginpoel,
36 : : const tk::UnsMesh::CoordMap& coordmap,
37 : : const tk::UnsMesh::Chunk& el,
38 : : const std::map< int, std::vector< std::size_t > >& bface,
39 : : const std::vector< std::size_t >& triinpoel,
40 : : const std::map< int, std::vector< std::size_t > >& bnode,
41 : : const std::unordered_map< std::size_t, std::set< std::size_t > >&
42 : : elemblockid,
43 : 1943 : int nchare ) :
44 : : m_meshid( meshid ),
45 : : m_host( transporter ),
46 : : m_meshwriter( meshwriter ),
47 : : m_cbs( cbs ),
48 : : m_scheme( scheme ),
49 : : m_reorderRefiner( reorderRefiner ),
50 : : m_ginpoel( ginpoel ),
51 : : m_coordmap( coordmap ),
52 : : m_el( el ),
53 : : m_nbnd( 0 ),
54 : : m_bface( bface ),
55 : : m_triinpoel( triinpoel ),
56 : : m_bnode( bnode ),
57 : : m_elemblockid( elemblockid ),
58 : : m_nchare( nchare ),
59 : : m_nodeset( begin(ginpoel), end(ginpoel) ),
60 : : m_noffset( 0 ),
61 : : m_nodech(),
62 : : m_chnode(),
63 : : m_edgech(),
64 : : m_chedge(),
65 : : m_msum(),
66 : : m_reordcomm(),
67 : : m_start( 0 ),
68 : : m_newnodes(),
69 : : m_newcoordmap(),
70 : : m_reqnodes(),
71 : : m_lower( 0 ),
72 [ + - ][ + - ]: 1943 : m_upper( 0 )
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ]
73 : : // *****************************************************************************
74 : : // Constructor: prepare owned mesh node IDs for reordering
75 : : //! \param[in] meshid Mesh ID
76 : : //! \param[in] transporter Transporter (host) Charm++ proxy
77 : : //! \param[in] meshwriter Mesh writer Charm++ proxy
78 : : //! \param[in] cbs Charm++ callbacks for Sorter
79 : : //! \param[in] scheme Discretization schemes (one per mesh)
80 : : //! \param[in] reorderRefiner Callback to use to send reordered mesh to Refiner
81 : : //! \param[in] ginpoel Mesh connectivity (this chare) using global node IDs
82 : : //! \param[in] coordmap Mesh node coordinates (this chare) for global node IDs
83 : : //! \param[in] bface Face lists mapped to side set ids
84 : : //! \param[in] triinpoel Interconnectivity of points and boundary-faces
85 : : //! \param[in] bnode Node ids mapped to side set ids
86 : : //! \param[in] elemblockid Local tet ids associated to mesh block ids
87 : : //! \param[in] nchare Total number of Charm++ worker chares
88 : : // *****************************************************************************
89 : : {
90 : : // Ensure boundary face ids will not index out of face connectivity
91 [ + - ][ - + ]: 184585 : Assert( std::all_of( begin(m_bface), end(m_bface),
[ - - ][ - - ]
[ - - ]
92 : : [&](const auto& s)
93 : : { return std::all_of( begin(s.second), end(s.second),
94 : : [&](auto f){ return f*3+2 < m_triinpoel.size(); } ); } ),
95 : : "Boundary face data structures inconsistent" );
96 : 1943 : }
97 : :
98 : : void
99 : 1943 : Sorter::setup( std::size_t npoin )
100 : : // *****************************************************************************
101 : : // Setup chare mesh boundary node communication map
102 : : //! \param[in] npoin Total number of mesh points in mesh. Note that the number
103 : : //! of mesh points does not have to be exactly the total number of points in
104 : : //! the mesh. It can be a larger number, but not less. This is only used here
105 : : //! to assign nodes to workers that will assign ids to mesh nodes during node
106 : : //! reordering.
107 : : // *****************************************************************************
108 : : {
109 : : // Compute the number of nodes (chunksize) a chare will build a node
110 : : // communication map for. We compute two values of chunksize: one for when
111 : : // the global node ids are abounded between [0...npoin-1], inclusive, and
112 : : // another one for when the global node ids are assigned by a hash algorithm
113 : : // during initial mesh refinement. In the latter case, the maximum
114 : : // representable value of a std::size_t is assumed to be the large global node
115 : : // id and is used to compute the chunksize. To compute the bin id, we attempt
116 : : // to use the first chunksize first: if it gives a chare id that is
117 : : // (strictly) lower than the number of chares, that's good. If not, we compute
118 : : // the bin id based on the second chunksize, which almost always will give a
119 : : // bin id strictly lower than the number of chares, except if the global node
120 : : // id assigned by the hash algorithm in Refiner hits the maximum
121 : : // representable number in std::size_t. If that is the case, we just assign
122 : : // that node to the last chare.
123 : 1943 : auto N = static_cast< std::size_t >( m_nchare );
124 : : std::array< std::size_t, 2 > chunksize{{
125 : 1943 : npoin / N, std::numeric_limits< std::size_t >::max() / N }};
126 : :
127 : 1943 : const auto scheme = g_inputdeck.get< tag::scheme >();
128 : :
129 : : // Find chare-boundary nodes and edges of our mesh chunk. This algorithm
130 : : // collects the global mesh node ids and edges on the chare boundary. A node
131 : : // is on a chare boundary if it belongs to a face of a tetrahedron that has
132 : : // no neighbor tet at a face. The edge is on the chare boundary if its first
133 : : // edge-end point is on a chare boundary. The nodes are categorized to bins
134 : : // that will be sent to different chares to build point-to-point
135 : : // communication maps across all chares. The binning is determined by the
136 : : // global node id divided by the chunksizes. See discussion above on how we
137 : : // use two chunksizes for global node ids assigned by the hash algorithm in
138 : : // Refiner (if initial mesh refinement has been done).
139 : 3886 : tk::CommMaps chbnd;
140 [ + - ]: 3886 : auto el = tk::global2local( m_ginpoel ); // generate local mesh data
141 : 1943 : const auto& inpoel = std::get< 0 >( el ); // local connectivity
142 [ + - ]: 3886 : auto esup = tk::genEsup( inpoel, 4 ); // elements surrounding points
143 [ + - ]: 3886 : auto esuel = tk::genEsuelTet( inpoel, esup ); // elems surrounding elements
144 [ + + ]: 546990 : for (std::size_t e=0; e<esuel.size()/4; ++e) {
145 : 545047 : auto mark = e*4;
146 [ + + ]: 2725235 : for (std::size_t f=0; f<4; ++f)
147 [ + + ]: 2180188 : if (esuel[mark+f] == -1)
148 [ + + ]: 1223400 : for (std::size_t n=0; n<3; ++n) {
149 : 917550 : auto g = m_ginpoel[ mark+tk::lpofa[f][n] ];
150 : 917550 : auto bin = g / chunksize[0];
151 [ + + ]: 917550 : if (bin >= N) bin = g / chunksize[1];
152 [ - + ]: 917550 : if (bin >= N) bin = N - 1;
153 [ - + ][ - - ]: 917550 : Assert( bin < N, "Will index out of number of chares" );
[ - - ][ - - ]
154 [ + - ]: 917550 : auto& b = chbnd[ static_cast< int >( bin ) ];
155 [ + - ]: 917550 : b.get< tag::node >().insert( g );
156 [ + + ][ + + ]: 917550 : if (scheme == ctr::SchemeType::ALECG ||
157 : : scheme == ctr::SchemeType::OversetFE) {
158 : 436512 : auto h = m_ginpoel[ mark + tk::lpofa[ f ][ tk::lpoet[n][1] ] ];
159 [ + - ]: 436512 : b.get< tag::edge >().insert( { std::min(g,h), std::max(g,h) } );
160 : : }
161 : : }
162 : : }
163 : :
164 : : // Send boundary data in bins to chares that will compute communication maps
165 : : // for the data in the bin. These bins form a distributed table. Note that
166 : : // we only send data to those chares that have data to work on. The receiving
167 : : // sides do not know in advance if they receive messages or not. Completion
168 : : // is detected by having the receiver respond back and counting the responses
169 : : // on the sender side, i.e., this chare.
170 : 1943 : m_nbnd = chbnd.size();
171 [ - + ]: 1943 : if (m_nbnd == 0)
172 [ - - ]: 0 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
173 : 0 : m_cbs.get< tag::queried >() );
174 : : else
175 [ + + ]: 14227 : for (const auto& [ targetchare, bnd ] : chbnd)
176 [ + - ][ + - ]: 12284 : thisProxy[ targetchare ].query( thisIndex, bnd );
177 : 1943 : }
178 : :
179 : : void
180 : 12284 : Sorter::query( int fromch, const tk::AllCommMaps& bnd )
181 : : // *****************************************************************************
182 : : // Incoming query for a list of mesh nodes for which this chare compiles node
183 : : // communication maps
184 : : //! \param[in] fromch Sender chare ID
185 : : //! \param[in] bnd Chare-boundary data from another chare
186 : : // *****************************************************************************
187 : : {
188 : : // Store incoming nodes in node->chare and its inverse, chare->node, maps
189 : 12284 : const auto& nodes = bnd.get< tag::node >();
190 [ + + ][ + - ]: 163596 : for (auto n : nodes) m_nodech[ n ].push_back( fromch );
[ + - ]
191 : 12284 : m_chnode[ fromch ].insert( begin(nodes), end(nodes) );
192 : :
193 : : // Store incoming edges in edge->chare and its inverse, chare->edge, maps
194 : 12284 : const auto& edges = bnd.get< tag::edge >();
195 [ + + ][ + - ]: 301954 : for (const auto& e : edges) m_edgech[ e ].push_back( fromch );
[ + - ]
196 : 12284 : m_chedge[ fromch ].insert( begin(edges), end(edges) );
197 : :
198 : : // Report back to chare message received from
199 [ + - ]: 12284 : thisProxy[ fromch ].recvquery();
200 : 12284 : }
201 : :
202 : : void
203 : 12284 : Sorter::recvquery()
204 : : // *****************************************************************************
205 : : // Receive receipt of boundary node lists to query
206 : : // *****************************************************************************
207 : : {
208 [ + + ]: 12284 : if (--m_nbnd == 0)
209 : 1943 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
210 : 1943 : m_cbs.get< tag::queried >() );
211 : 12284 : }
212 : :
213 : : void
214 : 1943 : Sorter::response()
215 : : // *****************************************************************************
216 : : // Respond to boundary node list queries
217 : : // *****************************************************************************
218 : : {
219 : 3886 : std::unordered_map< int, tk::CommMaps > exp;
220 : :
221 : : // Compute node communication map to be sent back to chares
222 [ + + ]: 14227 : for (const auto& [ neighborchare, bndnodes ] : m_chnode) {
223 [ + - ]: 12284 : auto& nc = exp[ neighborchare ];
224 [ + + ]: 163596 : for (auto n : bndnodes)
225 [ + - ][ + + ]: 429248 : for (auto d : tk::cref_find(m_nodech,n))
226 [ + + ]: 277936 : if (d != neighborchare)
227 [ + - ][ + - ]: 126624 : nc[d].get< tag::node >().insert( n );
228 : : }
229 : :
230 : : // Compute edge communication map to be sent back to chares
231 [ + + ]: 14227 : for (const auto& [ neighborchare, bndedges ] : m_chedge) {
232 [ + - ]: 12284 : auto& ec = exp[ neighborchare ];
233 [ + + ]: 301954 : for (const auto& e : bndedges)
234 [ + - ][ + + ]: 761228 : for (auto d : tk::cref_find(m_edgech,e))
235 [ + + ]: 471558 : if (d != neighborchare)
236 [ + - ][ + - ]: 181888 : ec[d].get< tag::edge >().insert( e );
237 : : }
238 : :
239 : : // Send communication maps to chares that issued a query to us. Communication
240 : : // maps were computed above for those chares that queried this map from us.
241 : : // This data form a distributed table and we only work on a chunk of it. Note
242 : : // that we only send data back to those chares that have queried us. The
243 : : // receiving sides do not know in advance if the receive messages or not.
244 : : // Completion is detected by having the receiver respond back and counting
245 : : // the responses on the sender side, i.e., this chare.
246 : 1943 : m_nbnd = exp.size();
247 [ + + ]: 1943 : if (m_nbnd == 0)
248 [ + - ]: 932 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
249 : 932 : m_cbs.get< tag::responded >() );
250 : : else
251 [ + + ]: 13295 : for (const auto& [ targetchare, maps ] : exp)
252 [ + - ][ + - ]: 12284 : thisProxy[ targetchare ].bnd( thisIndex, maps );
253 : 1943 : }
254 : :
255 : : void
256 : 12284 : Sorter::bnd( int fromch, const tk::CommMaps& msum )
257 : : // *****************************************************************************
258 : : // Receive boundary node communication maps for our mesh chunk
259 : : //! \param[in] fromch Sender chare ID
260 : : //! \param[in] msum Communication map(s) assembled by chare fromch
261 : : // *****************************************************************************
262 : : {
263 [ + + ]: 65274 : for (const auto& [ neighborchare, maps ] : msum) {
264 [ + - ]: 52990 : auto& m = m_msum[ neighborchare ];
265 : 52990 : const auto& nodemap = maps.get< tag::node >();
266 [ + - ]: 52990 : m.get< tag::node >().insert( begin(nodemap), end(nodemap) );
267 : 52990 : const auto& edgemap = maps.get< tag::edge >();
268 [ + - ]: 52990 : m.get< tag::edge >().insert( begin(edgemap), end(edgemap) );
269 : : }
270 : :
271 : : // Report back to chare message received from
272 [ + - ]: 12284 : thisProxy[ fromch ].recvbnd();
273 : 12284 : }
274 : :
275 : : void
276 : 12284 : Sorter::recvbnd()
277 : : // *****************************************************************************
278 : : // Receive receipt of boundary node communication map
279 : : // *****************************************************************************
280 : : {
281 [ + + ]: 12284 : if (--m_nbnd == 0)
282 : 1011 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
283 : 1011 : m_cbs.get< tag::responded >() );
284 : 12284 : }
285 : :
286 : : void
287 : 1943 : Sorter::start()
288 : : // *****************************************************************************
289 : : // Start reordering (if enabled)
290 : : // *****************************************************************************
291 : : {
292 : : // Keep only those edges in edge comm map whose both end-points are in the
293 : : // node comm map
294 [ + + ]: 21595 : for (auto& [ neighborchare, maps ] : m_msum) {
295 : 19652 : const auto& nodes = maps.get< tag::node >();
296 : 39304 : tk::EdgeSet edges;
297 [ + + ]: 134258 : for (const auto& e : maps.get< tag::edge >())
298 [ + - ][ + - ]: 114606 : if (nodes.find(e[0]) != end(nodes) && nodes.find(e[1]) != end(nodes))
[ + - ][ + - ]
[ + - ]
299 [ + - ]: 114606 : edges.insert( e );
300 : 19652 : maps.get< tag::edge >() = std::move(edges);
301 : : }
302 : :
303 [ - + ]: 1943 : if (g_inputdeck.get< tag::cmd, tag::feedback >()) m_host.chcomm();
304 : :
305 : 1943 : tk::destroy( m_nodech );
306 : 1943 : tk::destroy( m_chnode );
307 : :
308 [ + + ]: 1943 : if (g_inputdeck.get< tag::pelocal_reorder >())
309 : 18 : mask(); // continue with mesh node reordering if requested (or required)
310 : : else
311 : 1925 : createDiscWorkers(); // skip mesh node reordering
312 : 1943 : }
313 : :
314 : : void
315 : 18 : Sorter::mask()
316 : : // *****************************************************************************
317 : : // Start preparing for mesh node reordering in parallel
318 : : // *****************************************************************************
319 : : {
320 : : // Compute asymmetric communcation map that will be used for reordering. This
321 : : // communication map is asymmetric because it associates global mesh node IDs
322 : : // to chares only with lower IDs than thisIndex. That is because this chare
323 : : // will need to receive new (reorderd) node IDs only from chares with lower
324 : : // IDs than thisIndex during node reordering. Since it only stores data for
325 : : // lower chare IDs, it is asymmetric. Note that because of this algorithm the
326 : : // type of m_msum is an ordered map, because of the std::none_of() algorithm
327 : : // needs to look at ALL chares this chare potentially communicates nodes with
328 : : // that have lower chare IDs that thisIndex. Since the map is ordered, it can
329 : : // walk through from the beginning of m_msum until the outer loop variable c,
330 : : // which is the chare ID the outer loop works on in a given cycle.
331 [ + + ]: 30 : for (auto c=m_msum.cbegin(); c!=m_msum.cend(); ++c)
332 [ + + ]: 12 : if (thisIndex > c->first) {
333 [ + - ]: 6 : auto& n = m_reordcomm[ c->first ];
334 [ + + ]: 307 : for (auto j : c->second.get< tag::node >())
335 [ + - ][ + - ]: 301 : if (std::none_of( m_msum.cbegin(), c,
336 : 0 : [j]( const auto& s ) {
337 : 0 : const auto& nodemap = s.second.template get< tag::node >();
338 [ - - ]: 0 : return nodemap.find(j) != end(nodemap); } ))
339 : : {
340 [ + - ]: 301 : n.insert(j);
341 : : }
342 [ - + ][ - - ]: 6 : if (n.empty()) m_reordcomm.erase( c->first );
343 : : }
344 : :
345 : : // Count up total number of nodes this chare will need to receive
346 : 18 : auto nrecv = tk::sumvalsize( m_reordcomm );
347 : :
348 [ - + ][ - - ]: 18 : if ( g_inputdeck.get< tag::cmd, tag::feedback >() ) m_host.chmask();
349 : :
350 : : // Compute number of mesh node IDs we will assign IDs to
351 : 18 : auto nuniq = m_nodeset.size() - nrecv;
352 : :
353 : : // Start computing offsets for node reordering
354 [ + - ]: 18 : thisProxy.offset( thisIndex, nuniq );
355 : 18 : }
356 : :
357 : : void
358 : 30 : Sorter::offset( int c, std::size_t u )
359 : : // *****************************************************************************
360 : : // Receive number of uniquely assigned global mesh node IDs from chares with
361 : : // lower IDs than thisIndex
362 : : //! \param[in] c Chare ID
363 : : //! \param[in] u Number of mesh node IDs chare c will assign IDs to
364 : : //! \details This function computes the offset each chare will need to start
365 : : //! assigning its new node IDs from. The offset for a chare is the
366 : : //! offset for the previous chare plus the number of node IDs the previous
367 : : //! chare (uniquely) assigns new IDs for minus the number of node IDs the
368 : : //! previous chare receives from others (lower chares). This is computed here
369 : : //! in a parallel/distributed fashion by each chare sending its number of node
370 : : //! IDs (that it uniquely assigns) to all chares. Note that each chare would
371 : : //! only need to send this information to chares with higher IDs, but instead
372 : : //! this function is called in a broadcast fashion, because that is more
373 : : //! efficient than individual calls to only chares with higher IDs. Therefore
374 : : //! when computing the offsets, we only count the lower chares. When this is
375 : : //! done, we have the precise asymmetric communication map as well as the
376 : : //! start offset on all chares and so we can start the distributed global mesh
377 : : //! node ID reordering.
378 : : // *****************************************************************************
379 : : {
380 [ + + ]: 30 : if (c < thisIndex) m_start += u;
381 [ + + ]: 30 : if (++m_noffset == m_nchare) reorder();
382 : 30 : }
383 : :
384 : : void
385 : 18 : Sorter::reorder()
386 : : // *****************************************************************************
387 : : // Reorder global mesh node IDs
388 : : // *****************************************************************************
389 : : {
390 : : // Activate SDAG waits for arriving requests from other chares requesting new
391 : : // node IDs for node IDs we assign new IDs to during reordering; and for
392 : : // computing/receiving lower and upper bounds of global node IDs our chare's
393 : : // linear system will operate on after reordering.
394 [ + - ][ + - ]: 18 : thisProxy[ thisIndex ].wait4prep();
395 : :
396 : : // Send out request for new global node IDs for nodes we do not reorder
397 [ + + ]: 24 : for (const auto& [ targetchare, nodes ] : m_reordcomm)
398 [ + - ][ + - ]: 6 : thisProxy[ targetchare ].request( thisIndex, nodes );
399 : :
400 : : // Lambda to decide if node is assigned a new ID by this chare. If node is not
401 : : // found in the asymmetric communication map, it is owned, i.e., this chare
402 : : // assigns its new id.
403 : 24778 : auto ownnode = [ this ]( std::size_t p ) {
404 : 12389 : return std::all_of( m_reordcomm.cbegin(), m_reordcomm.cend(),
405 : 2344 : [&](const auto& s)
406 [ + - ]: 14733 : { return s.second.find(p) == s.second.cend(); } );
407 : 18 : };
408 : :
409 : : // Reorder our chunk of the mesh node IDs. Looping through all of our node
410 : : // IDs, we test if we are to assign a new ID to a node ID, and if so, we
411 : : // assign a new ID, i.e., reorder, by constructing a map associating new to
412 : : // old IDs (m_newnodes). We also count up the reordered nodes, which serves as
413 : : // the new node id. We also store the node coordinates associated to the new
414 : : // node ID.
415 [ + + ]: 12407 : for (auto p : m_nodeset)
416 [ + - ][ + + ]: 12389 : if (ownnode(p)) {
417 [ + - ]: 12088 : m_newnodes[ p ] = m_start; // assign new node ID (reorder)
418 [ + - ][ + - ]: 12088 : m_newcoordmap.emplace( m_start, tk::cref_find(m_coordmap,p) );
419 : 12088 : ++m_start;
420 : : }
421 : :
422 : : // Trigger SDAG wait indicating that reordering our node IDs are complete
423 [ + - ]: 18 : reorderowned_complete();
424 : :
425 : : // If all our nodes have new IDs assigned, reordering complete on this chare
426 [ + + ][ + - ]: 18 : if (m_newnodes.size() == m_nodeset.size()) finish();
427 : 18 : }
428 : :
429 : : void
430 : 6 : Sorter::request( int c, const std::unordered_set< std::size_t >& nd )
431 : : // *****************************************************************************
432 : : // Request new global node IDs for old node IDs
433 : : //! \param[in] c Chare request coming from and to which we send new IDs to
434 : : //! \param[in] nd Set of old node IDs whose new IDs are requested
435 : : // *****************************************************************************
436 : : {
437 : : // Queue up requesting chare and node IDs
438 [ + - ]: 6 : m_reqnodes.push_back( { c, nd } );
439 : : // Trigger SDAG wait signaling that node IDs have been requested from us
440 : 6 : nodes_requested_complete();
441 : 6 : }
442 : :
443 : : void
444 : 6 : Sorter::prepare()
445 : : // *****************************************************************************
446 : : // Find new node IDs for old ones and return them to the requestor(s)
447 : : // *****************************************************************************
448 : : {
449 : : // Find and return new node IDs to sender
450 [ + + ]: 12 : for (const auto& [ requestorchare, nodes ] : m_reqnodes) {
451 : : std::unordered_map< std::size_t,
452 : 6 : std::tuple< std::size_t, tk::UnsMesh::Coord > > n;
453 [ + + ]: 307 : for (auto p : nodes) {
454 [ + - ]: 301 : auto newid = tk::cref_find( m_newnodes, p );
455 : : n.emplace( p,
456 [ + - ][ + - ]: 301 : std::make_tuple( newid, tk::cref_find(m_newcoordmap,newid) ) );
[ + - ]
457 : : }
458 [ + - ][ + - ]: 6 : thisProxy[ requestorchare ].neworder( n );
459 : : }
460 : :
461 : 6 : tk::destroy( m_reqnodes ); // Clear queue of requests just fulfilled
462 : :
463 : : // Re-enable SDAG wait for preparing new node requests
464 [ + - ]: 6 : thisProxy[ thisIndex ].wait4prep();
465 : :
466 : : // Re-enable trigger signaling that reordering of owned node IDs are
467 : : // complete right away
468 : 6 : reorderowned_complete();
469 : 6 : }
470 : :
471 : : void
472 : 6 : Sorter::neworder( const std::unordered_map< std::size_t,
473 : : std::tuple< std::size_t, tk::UnsMesh::Coord > >& nodes )
474 : : // *****************************************************************************
475 : : // Receive new (reordered) global node IDs
476 : : //! \param[in] nodes Map associating new to old node IDs
477 : : // *****************************************************************************
478 : : {
479 : : // Store new node IDs associated to old ones, and node coordinates associated
480 : : // to new node IDs.
481 [ + + ]: 307 : for (const auto& [ oldid, newnodes ] : nodes) {
482 : 301 : auto newid = std::get< 0 >( newnodes );
483 [ + - ]: 301 : m_newnodes[ oldid ] = newid;
484 [ + - ]: 301 : m_newcoordmap.emplace( newid, std::get< 1 >( newnodes ) );
485 : : }
486 : :
487 : : // If all our nodes have new IDs assigned, reorder complete on this PE
488 [ + - ]: 6 : if (m_newnodes.size() == m_nodeset.size()) finish();
489 : 6 : }
490 : :
491 : : void
492 : 18 : Sorter::finish()
493 : : // *****************************************************************************
494 : : // Compute final result of reordering
495 : : //! \details Reordering is now complete on this chare. We now remap all mesh
496 : : //! data to reflect the new ordering.
497 : : // *****************************************************************************
498 : : {
499 : : // Update elem connectivity with the reordered node IDs
500 : 18 : tk::remap( m_ginpoel, m_newnodes );
501 : :
502 : : // Update node coordinate map with the reordered IDs
503 : 18 : m_coordmap = m_newcoordmap;
504 : :
505 : : // Update mesh chunk data structure held in our state with new node order
506 : 18 : m_el = tk::global2local( m_ginpoel );
507 : :
508 : : // Update symmetric chare-node communication map with the reordered IDs
509 [ + + ]: 30 : for (auto& [ neighborchare, maps ] : m_msum) {
510 : :
511 : 24 : tk::NodeSet n;
512 [ + + ]: 614 : for (auto p : maps.get< tag::node >())
513 [ + - ][ + - ]: 602 : n.insert( tk::cref_find( m_newnodes, p ) );
514 : 12 : maps.get< tag::node >() = std::move( n );
515 : :
516 : 24 : tk::EdgeSet e;
517 [ + + ]: 1444 : for (const auto& ed : maps.get< tag::edge >()) {
518 [ + - ]: 1432 : e.insert( { tk::cref_find(m_newnodes,ed[0]),
519 [ + - ][ + - ]: 1432 : tk::cref_find(m_newnodes,ed[1]) } );
520 : : }
521 : 12 : maps.get< tag::edge >() = std::move( e );
522 : :
523 : : }
524 : :
525 : : // Update boundary face-node connectivity with the reordered node IDs
526 : 18 : tk::remap( m_triinpoel, m_newnodes );
527 : :
528 : : // Update boundary node lists with the reordered node IDs
529 [ - + ][ - - ]: 18 : for (auto& [ setid, nodes ] : m_bnode) tk::remap( nodes, m_newnodes );
530 : :
531 : : // Update mesh in Refiner after reordering
532 : 18 : m_reorderRefiner.send();
533 : :
534 : : // Progress report to host
535 [ - + ]: 18 : if ( g_inputdeck.get< tag::cmd, tag::feedback >() ) m_host.chreordered();
536 : :
537 : 18 : createDiscWorkers();
538 : 18 : }
539 : :
540 : : void
541 : 18 : Sorter::mesh( std::vector< std::size_t >& ginpoel,
542 : : tk::UnsMesh::CoordMap& coordmap,
543 : : std::vector< std::size_t >& triinpoel,
544 : : std::map< int, std::vector< std::size_t > >& bnode )
545 : : // *****************************************************************************
546 : : // Update mesh data we hold for whoever calls this function
547 : : //! \param[in,out] ginpoel Mesh connectivity using global IDs
548 : : //! \param[in,out] coordmap Map of mesh node coordinates
549 : : //! \param[in,out] triinpoel Boundary face-node connectivity
550 : : //! \param[in] bnode Node lists of side sets
551 : : // *****************************************************************************
552 : : {
553 : 18 : ginpoel = m_ginpoel;
554 : 18 : coordmap = m_coordmap;
555 : 18 : triinpoel = m_triinpoel;
556 : 18 : bnode = m_bnode;
557 : 18 : }
558 : :
559 : : void
560 : 1943 : Sorter::createDiscWorkers()
561 : : // *****************************************************************************
562 : : // Create Discretization chare array elements on this PE
563 : : //! \details We create chare array elements by calling the insert() member
564 : : //! function, which allows specifying the PE on which the array element is
565 : : //! created. and we send each chare array element the chunk of mesh it will
566 : : //! operate on.
567 : : // *****************************************************************************
568 : : {
569 : 3886 : std::vector< CProxy_Discretization > disc;
570 [ + + ][ + - ]: 3900 : for (auto& d : m_scheme) disc.push_back( d.disc() );
571 : :
572 : : // Create worker array element using Charm++ dynamic chare array element
573 : : // insertion: last arg: PE chare is created on. See also Charm++ manual, Sec.
574 : : // "Dynamic Insertion".
575 : :
576 [ + - ][ + - ]: 3886 : m_scheme[m_meshid].disc()[ thisIndex ].insert( m_meshid, disc,
577 : 1943 : m_scheme[m_meshid].ale(),
578 : 1943 : m_scheme[m_meshid].conjugategradients(), m_host, m_meshwriter, m_coordmap,
579 : 1943 : m_el, m_msum, m_bface, m_triinpoel, m_elemblockid, m_nchare );
580 : :
581 [ + - ]: 1943 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
582 : 1943 : m_cbs.get< tag::discinserted >() );
583 : 1943 : }
584 : :
585 : : void
586 : 1943 : Sorter::createWorkers()
587 : : // *****************************************************************************
588 : : // Create worker chare array element
589 : : // *****************************************************************************
590 : : {
591 : : // Make sure (bound) base is already created and accessible
592 [ + - ][ - + ]: 1943 : Assert( m_scheme[m_meshid].disc()[thisIndex].ckLocal() != nullptr,
[ - - ][ - - ]
[ - - ]
593 : : "About to pass nullptr" );
594 : :
595 : : // Create worker array element using Charm++ dynamic chare array element
596 : : // insertion: 1st arg: chare id, other args: Discretization's child ctor args.
597 : : // See also Charm++ manual, Sec. "Dynamic Insertion".
598 : :
599 [ + - ]: 3886 : m_scheme[m_meshid].insert( thisIndex, m_scheme[m_meshid].disc(),
600 : 1943 : m_scheme[m_meshid].ghosts(), m_bface, m_bnode, m_triinpoel );
601 : :
602 [ - + ]: 1943 : if ( g_inputdeck.get< tag::cmd, tag::feedback >() ) m_host.chcreated();
603 : :
604 : 1943 : contribute( sizeof(std::size_t), &m_meshid, CkReduction::nop,
605 : 1943 : m_cbs.get< tag::workinserted >() );
606 : :
607 : : // Free up some memory
608 : 1943 : tk::destroy( m_ginpoel );
609 : 1943 : tk::destroy( m_coordmap );
610 : 1943 : tk::destroy( m_bface );
611 : 1943 : tk::destroy( m_triinpoel );
612 : 1943 : tk::destroy( m_elemblockid );
613 : 1943 : tk::destroy( m_bnode );
614 : 1943 : tk::destroy( m_nodeset );
615 : 1943 : tk::destroy( m_nodech );
616 : 1943 : tk::destroy( m_chnode );
617 : 1943 : tk::destroy( m_msum );
618 : 1943 : tk::destroy( m_reordcomm );
619 : 1943 : tk::destroy( m_newnodes );
620 : 1943 : tk::destroy( m_reqnodes );
621 : 1943 : }
622 : :
623 : : #include "NoWarning/sorter.def.h"
|
