Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/IO/MeshWriter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Charm++ group for outputing mesh data to file
9 : : \details Charm++ group definition used to output data associated to
10 : : unstructured meshes to file(s). Charm++ chares (work units) send mesh and
11 : : field data associated to mesh entities to the MeshWriter class defined here
12 : : to write the data to file(s).
13 : : */
14 : : // *****************************************************************************
15 : :
16 : : #include "QuinoaBuildConfig.hpp"
17 : : #include "MeshWriter.hpp"
18 : : #include "Reorder.hpp"
19 : : #include "ExodusIIMeshWriter.hpp"
20 : :
21 : : #ifdef HAS_ROOT
22 : : #include "RootMeshWriter.hpp"
23 : : #endif
24 : :
25 : : using tk::MeshWriter;
26 : :
27 : 666 : MeshWriter::MeshWriter( ctr::FieldFileType filetype,
28 : : Centering bnd_centering,
29 : : bool benchmark,
30 : 666 : std::size_t nmesh ) :
31 : : m_filetype( filetype ),
32 : : m_bndCentering( bnd_centering ),
33 : : m_benchmark( benchmark ),
34 : : m_nchare( 0 ),
35 : 666 : m_nmesh( nmesh )
36 : : // *****************************************************************************
37 : : // Constructor: set some defaults that stay constant at all times
38 : : //! \param[in] filetype Output file format type
39 : : //! \param[in] bnd_centering Centering to identify what boundary data to write.
40 : : //! For a nodal scheme, e.g., DiagCG, this is nodal, for a DG scheme, this is
41 : : //! cell-based.
42 : : //! \param[in] benchmark True of benchmark mode. No field output happens in
43 : : //! benchmark mode. This (and associated if tests) are here so client code
44 : : //! does not have to deal with this.
45 : : //! \param[in] nmesh Total number of meshes
46 : : // *****************************************************************************
47 : : {
48 : 666 : }
49 : :
50 : : void
51 : 666 : MeshWriter::nchare( int n )
52 : : // *****************************************************************************
53 : : // Set the total number of chares
54 : : //! \param[in] n Total number of chares across the whole problem
55 : : // *****************************************************************************
56 : : {
57 : 666 : m_nchare = n;
58 : 666 : }
59 : :
60 : : void
61 : 6570 : MeshWriter::write(
62 : : std::size_t meshid,
63 : : bool meshoutput,
64 : : bool fieldoutput,
65 : : uint64_t itr,
66 : : uint64_t itf,
67 : : tk::real time,
68 : : int chareid,
69 : : const std::string& basefilename,
70 : : const std::vector< std::size_t >& inpoel,
71 : : const UnsMesh::Coords& coord,
72 : : const std::map< int, std::vector< std::size_t > >& bface,
73 : : const std::map< int, std::vector< std::size_t > >& bnode,
74 : : const std::vector< std::size_t >& triinpoel,
75 : : const std::vector< std::string >& elemfieldnames,
76 : : const std::vector< std::string >& nodefieldnames,
77 : : const std::vector< std::string >& nodesurfnames,
78 : : const std::vector< std::vector< tk::real > >& elemfields,
79 : : const std::vector< std::vector< tk::real > >& nodefields,
80 : : const std::vector< std::vector< tk::real > >& nodesurfs,
81 : : const std::set< int >& outsets,
82 : : CkCallback c )
83 : : // *****************************************************************************
84 : : // Output unstructured mesh into file
85 : : //! \param[in] meshid Mesh Id
86 : : //! \param[in] meshoutput True if mesh is to be written
87 : : //! \param[in] fieldoutput True if field data is to be written
88 : : //! \param[in] itr Iteration count since a new mesh. New mesh in this context
89 : : //! means that either the mesh is moved and/or its topology has changed.
90 : : //! \param[in] itf Field output iteration count
91 : : //! \param[in] time Physical time this at this field output dump
92 : : //! \param[in] chareid The chare id the write-to-file request is coming from
93 : : //! \param[in] basefilename String to use as the base of the filename
94 : : //! \param[in] inpoel Mesh connectivity for the mesh chunk to be written with
95 : : //! local ids
96 : : //! \param[in] coord Node coordinates of the mesh chunk to be written
97 : : //! \param[in] bface Map of boundary-face lists mapped to corresponding side set
98 : : //! ids for this mesh chunk
99 : : //! \param[in] bnode Map of boundary-node lists mapped to corresponding side set
100 : : //! ids for this mesh chunk with local ids
101 : : //! \param[in] triinpoel Interconnectivity of points and boundary-face in this
102 : : //! mesh chunk with local ids
103 : : //! \param[in] elemfieldnames Names of element fields to be output to file
104 : : //! \param[in] nodefieldnames Names of node fields to be output to file
105 : : //! \param[in] nodesurfnames Names of node surface fields to be output to file
106 : : //! \param[in] elemfields Field data in mesh elements to output to file
107 : : //! \param[in] nodefields Field data in mesh nodes to output to file
108 : : //! \param[in] nodesurfs Surface field data in mesh nodes to output to file
109 : : //! \param[in] outsets Unique set of surface side set ids along which to save
110 : : //! solution field variables
111 : : //! \param[in] c Function to continue with after the write
112 : : // *****************************************************************************
113 : : {
114 [ + - ]: 6570 : if (!m_benchmark) {
115 : :
116 : : // Generate filenames for volume and surface field output
117 : 6570 : auto vf = filename( basefilename, meshid, itr, chareid );
118 : :
119 [ + + ]: 6570 : if (meshoutput) {
120 : : #ifdef HAS_ROOT
121 : : if (m_filetype == ctr::FieldFileType::ROOT) {
122 : :
123 : : RootMeshWriter rmw( vf, 0 );
124 : : rmw.writeMesh( UnsMesh( inpoel, coord ) );
125 : : rmw.writeNodeVarNames( nodefieldnames );
126 : :
127 : : } else
128 : : #endif
129 [ + - ]: 1609 : if (m_filetype == ctr::FieldFileType::EXODUSII) {
130 : :
131 : : // Write volume mesh and field names
132 [ + - ]: 3218 : ExodusIIMeshWriter ev( vf, ExoWriter::CREATE );
133 : : // Write chare mesh (do not write side sets in parallel)
134 [ + + ]: 1609 : if (m_nchare == 1) {
135 : :
136 [ + + ]: 129 : if (m_bndCentering == Centering::ELEM)
137 [ + - ]: 59 : ev.writeMesh( inpoel, coord, bface, triinpoel );
138 [ + - ]: 70 : else if (m_bndCentering == Centering::NODE)
139 [ + - ]: 70 : ev.writeMesh( inpoel, coord, bnode );
140 [ - - ][ - - ]: 0 : else Throw( "Centering not handled for writing mesh" );
[ - - ][ - - ]
[ - - ][ - - ]
141 : :
142 : : } else {
143 [ + - ]: 1480 : ev.writeMesh< 4 >( inpoel, coord );
144 : : }
145 [ + - ]: 1609 : ev.writeElemVarNames( elemfieldnames );
146 : : Assert( nodefieldnames.size() == nodefields.size(), "Size mismatch" );
147 [ + - ]: 1609 : ev.writeNodeVarNames( nodefieldnames );
148 : :
149 : : // Write surface meshes and surface variable field names
150 [ + + ]: 1677 : for (auto s : outsets) {
151 [ + - ]: 68 : auto sf = filename( basefilename, meshid, itr, chareid, s );
152 [ + - ]: 109 : ExodusIIMeshWriter es( sf, ExoWriter::CREATE );
153 : : auto b = bface.find(s);
154 [ + + ]: 68 : if (b == end(bface)) {
155 : : // If a side set does not exist on a chare, write out a
156 : : // connectivity for a single triangle with its node coordinates of
157 : : // zero. This is so the paraview series reader can load side sets
158 : : // distributed across multiple files. See also
159 : : // https://www.paraview.org/Wiki/Restarted_Simulation_Readers.
160 [ + - ][ + - ]: 54 : es.writeMesh< 3 >( std::vector< std::size_t >{1,2,3},
[ - - ]
161 [ + - ][ + - ]: 27 : UnsMesh::Coords{{ {{0,0,0}}, {{0,0,0}}, {{0,0,0}} }} );
[ + - ]
162 [ + - ]: 27 : es.writeNodeVarNames( nodesurfnames );
163 : 27 : continue;
164 : : }
165 : : std::vector< std::size_t > nodes;
166 [ + + ]: 3185 : for (auto f : b->second) {
167 [ + - ]: 3144 : nodes.push_back( triinpoel[f*3+0] );
168 [ + - ]: 3144 : nodes.push_back( triinpoel[f*3+1] );
169 [ + - ]: 3144 : nodes.push_back( triinpoel[f*3+2] );
170 : : }
171 [ + - ]: 41 : auto [inp,gid,lid] = tk::global2local( nodes );
172 [ + - ]: 41 : tk::unique( nodes );
173 : 41 : auto nnode = nodes.size();
174 : : UnsMesh::Coords scoord;
175 [ + - ]: 41 : scoord[0].resize( nnode );
176 [ + - ]: 41 : scoord[1].resize( nnode );
177 [ + - ]: 41 : scoord[2].resize( nnode );
178 : : std::size_t j = 0;
179 [ + + ]: 2424 : for (auto i : nodes) {
180 : 2383 : scoord[0][j] = coord[0][i];
181 : 2383 : scoord[1][j] = coord[1][i];
182 : 2383 : scoord[2][j] = coord[2][i];
183 : 2383 : ++j;
184 : : }
185 [ + - ]: 41 : es.writeMesh< 3 >( inp, scoord );
186 [ + - ]: 41 : es.writeNodeVarNames( nodesurfnames );
187 : : }
188 : :
189 : : }
190 : : }
191 : :
192 [ + - ]: 6570 : if (fieldoutput) {
193 : : #ifdef HAS_ROOT
194 : : if (m_filetype == ctr::FieldFileType::ROOT) {
195 : :
196 : : RootMeshWriter rw( vf, 1 );
197 : : rw.writeTimeStamp( itf, time );
198 : : int varid = 0;
199 : : for (const auto& v : nodefields) rw.writeNodeScalar( itf, ++varid, v );
200 : :
201 : : } else
202 : : #endif
203 [ + - ]: 6570 : if (m_filetype == ctr::FieldFileType::EXODUSII) {
204 : :
205 : : // Write volume variable fields
206 [ + - ]: 13140 : ExodusIIMeshWriter ev( vf, ExoWriter::OPEN );
207 [ + - ]: 6570 : ev.writeTimeStamp( itf, time );
208 : : // Write volume element variable fields
209 : : int varid = 0;
210 [ + + ][ + - ]: 30475 : for (const auto& v : elemfields) ev.writeElemScalar( itf, ++varid, v );
211 : : // Write volume node variable fields
212 : : varid = 0;
213 [ + + ][ + - ]: 37143 : for (const auto& v : nodefields) ev.writeNodeScalar( itf, ++varid, v );
214 : :
215 : : // Write surface node variable fields
216 : : std::size_t j = 0;
217 : 6570 : auto nvar = static_cast< int >( nodesurfnames.size() ) ;
218 [ + + ]: 6718 : for (auto s : outsets) {
219 [ + - ]: 148 : auto sf = filename( basefilename, meshid, itr, chareid, s );
220 [ + - ]: 242 : ExodusIIMeshWriter es( sf, ExoWriter::OPEN );
221 [ + - ]: 148 : es.writeTimeStamp( itf, time );
222 [ + + ]: 148 : if (bface.find(s) == end(bface)) {
223 : : // If a side set does not exist on a chare, write out a
224 : : // a node field for a single triangle with zeros. This is so the
225 : : // paraview series reader can load side sets distributed across
226 : : // multiple files. See also
227 : : // https://www.paraview.org/Wiki/Restarted_Simulation_Readers.
228 [ + + ][ + - ]: 378 : for (int i=1; i<=nvar; ++i) es.writeNodeScalar( itf, i, {0,0,0} );
[ + - ]
229 : 54 : continue;
230 : : }
231 [ + + ]: 658 : for (int i=1; i<=nvar; ++i)
232 [ + - ]: 564 : es.writeNodeScalar( itf, i, nodesurfs[j++] );
233 : : }
234 : :
235 : : }
236 : : }
237 : :
238 : : }
239 : :
240 : 6570 : c.send();
241 : 6570 : }
242 : :
243 : : std::string
244 : 6786 : MeshWriter::filename( const std::string& basefilename,
245 : : std::size_t meshid,
246 : : uint64_t itr,
247 : : int chareid,
248 : : int surfid ) const
249 : : // *****************************************************************************
250 : : // Compute filename
251 : : //! \param[in] basefilename String use as the base filename.
252 : : //! \param[in] meshid Mesh Id
253 : : //! \param[in] itr Iteration count since a new mesh. New mesh in this context
254 : : //! means that either the mesh is moved and/or its topology has changed.
255 : : //! \param[in] chareid The chare id the write-to-file request is coming from
256 : : //! \param[in] surfid Surface ID if computing a surface filename
257 : : //! \details We use a file naming convention for large field output data that
258 : : //! allows ParaView to glue multiple files into a single simulation output by
259 : : //! only loading a single file. The base filename is followed by ".e-s.",
260 : : //! which probably stands for Exodus Sequence, followed by 3 integers:
261 : : //! (1) {RS}: counts the number of "restart dumps", but we use this for
262 : : //! counting the number of outputs with a different mesh, e.g., due to
263 : : //! mesh refinement, thus if this first number is new the mesh is new
264 : : //! compared to the previous (first) number afer ".e-s.",
265 : : //! (2) {NP}: total number of partitions (workers, chares), this is more than
266 : : //! the number of PEs with nonzero virtualization (overdecomposition), and
267 : : //! (3) {RANK}: worker (chare) id.
268 : : //! Thus {RANK} does spatial partitioning, while {RS} partitions in time, but
269 : : //! a single {RS} id may contain multiple time steps, which equals to the
270 : : //! number of time steps at which field output is saved without refining the
271 : : //! mesh.
272 : : //! \return Filename computed
273 : : //! \see https://www.paraview.org/Wiki/Restarted_Simulation_Readers
274 : : // *****************************************************************************
275 : : {
276 [ + + ][ + - ]: 14004 : return basefilename + (surfid ? "-surf." + std::to_string(surfid) : "")
[ + - ][ + - ]
[ - + ][ + + ]
[ - + ][ - - ]
[ - - ][ - - ]
277 [ - + ][ - - ]: 27144 : + (m_nmesh > 1 ? '.' + std::to_string(meshid) : "")
[ + - ][ + - ]
[ - + ][ - + ]
[ - + ][ - - ]
[ - - ][ - - ]
278 [ + - ][ - + ]: 13572 : + ".e-s"
[ - - ]
279 [ + - ][ + - ]: 20358 : + '.' + std::to_string( itr ) // iteration count with new mesh
[ - + ][ - + ]
[ - + ][ - - ]
[ - - ][ - - ]
280 [ + - ][ + - ]: 27144 : + '.' + std::to_string( m_nchare ) // total number of workers
[ + - ][ - + ]
[ - + ][ - + ]
[ - - ][ - - ]
[ - - ]
281 [ + - ][ + - ]: 20358 : + '.' + std::to_string( chareid ) // new file per worker
[ - + ][ - - ]
282 : : #ifdef HAS_ROOT
283 : : + (m_filetype == ctr::FieldFileType::ROOT ? ".root" : "")
284 : : #endif
285 : : ;
286 : : }
287 : :
288 : : #include "NoWarning/meshwriter.def.h"
|
