Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/IO/MeshWriter.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Charm++ group for outputing mesh data to file
9 : : \details Charm++ group definition used to output data associated to
10 : : unstructured meshes to file(s). Charm++ chares (work units) send mesh and
11 : : field data associated to mesh entities to the MeshWriter class defined here
12 : : to write the data to file(s).
13 : : */
14 : : // *****************************************************************************
15 : :
16 : : #include "QuinoaBuildConfig.hpp"
17 : : #include "MeshWriter.hpp"
18 : : #include "Reorder.hpp"
19 : : #include "ExodusIIMeshWriter.hpp"
20 : :
21 : : using tk::MeshWriter;
22 : :
23 : 574 : MeshWriter::MeshWriter( ctr::FieldFileType filetype,
24 : : Centering bnd_centering,
25 : : bool benchmark,
26 : 574 : std::size_t nmesh ) :
27 : : m_filetype( filetype ),
28 : : m_bndCentering( bnd_centering ),
29 : : m_benchmark( benchmark ),
30 : : m_nmesh( nmesh ),
31 [ + - ]: 574 : m_nchare( nmesh, 0 )
32 : : // *****************************************************************************
33 : : // Constructor: set some defaults that stay constant at all times
34 : : //! \param[in] filetype Output file format type
35 : : //! \param[in] bnd_centering Centering to identify what boundary data to write.
36 : : //! For a nodal scheme, e.g., ALECG, this is nodal, for a DG scheme, this is
37 : : //! cell-based.
38 : : //! \param[in] benchmark True of benchmark mode. No field output happens in
39 : : //! benchmark mode. This (and associated if tests) are here so client code
40 : : //! does not have to deal with this.
41 : : //! \param[in] nmesh Total number of meshes
42 : : // *****************************************************************************
43 : : {
44 : 574 : }
45 : :
46 : : void
47 : 574 : MeshWriter::nchare( std::size_t meshid, int n )
48 : : // *****************************************************************************
49 : : // Set the total number of chares
50 : : //! \param[in] meshid Mesh whose number of chares to set
51 : : //! \param[in] n Total number of chares across the whole problem (for a mesh)
52 : : // *****************************************************************************
53 : : {
54 [ - + ][ - - ]: 574 : Assert( meshid < m_nchare.size(), "Size mismatch" );
[ - - ][ - - ]
55 : 574 : m_nchare[ meshid ] = n;
56 : 574 : }
57 : :
58 : : void
59 : 3706 : MeshWriter::write(
60 : : std::size_t meshid,
61 : : bool meshoutput,
62 : : bool fieldoutput,
63 : : uint64_t itr,
64 : : uint64_t itf,
65 : : tk::real time,
66 : : int chareid,
67 : : const std::string& basefilename,
68 : : const std::vector< std::size_t >& inpoel,
69 : : const UnsMesh::Coords& coord,
70 : : const std::map< int, std::vector< std::size_t > >& bface,
71 : : const std::map< int, std::vector< std::size_t > >& bnode,
72 : : const std::vector< std::size_t >& triinpoel,
73 : : const std::vector< std::string >& elemfieldnames,
74 : : const std::vector< std::string >& nodefieldnames,
75 : : const std::vector< std::string >& elemsurfnames,
76 : : const std::vector< std::string >& nodesurfnames,
77 : : const std::vector< std::vector< tk::real > >& elemfields,
78 : : const std::vector< std::vector< tk::real > >& nodefields,
79 : : const std::vector< std::vector< tk::real > >& elemsurfs,
80 : : const std::vector< std::vector< tk::real > >& nodesurfs,
81 : : const std::set< int >& outsets,
82 : : CkCallback c )
83 : : // *****************************************************************************
84 : : // Output unstructured mesh into file
85 : : //! \param[in] meshid Mesh Id
86 : : //! \param[in] meshoutput True if mesh is to be written
87 : : //! \param[in] fieldoutput True if field data is to be written
88 : : //! \param[in] itr Iteration count since a new mesh. New mesh in this context
89 : : //! means that either the mesh is moved and/or its topology has changed.
90 : : //! \param[in] itf Field output iteration count
91 : : //! \param[in] time Physical time this at this field output dump
92 : : //! \param[in] chareid The chare id the write-to-file request is coming from
93 : : //! \param[in] basefilename String to use as the base of the filename
94 : : //! \param[in] inpoel Mesh connectivity for the mesh chunk to be written with
95 : : //! local ids
96 : : //! \param[in] coord Node coordinates of the mesh chunk to be written
97 : : //! \param[in] bface Map of boundary-face lists mapped to corresponding side set
98 : : //! ids for this mesh chunk
99 : : //! \param[in] bnode Map of boundary-node lists mapped to corresponding side set
100 : : //! ids for this mesh chunk with local ids
101 : : //! \param[in] triinpoel Interconnectivity of points and boundary-face in this
102 : : //! mesh chunk with local ids
103 : : //! \param[in] elemfieldnames Names of element fields to be output to file
104 : : //! \param[in] nodefieldnames Names of node fields to be output to file
105 : : //! \param[in] elemsurfnames Names of elemental surface fields to be output to
106 : : //! file
107 : : //! \param[in] nodesurfnames Names of node surface fields to be output to file
108 : : //! \param[in] elemfields Field data in mesh elements to output to file
109 : : //! \param[in] nodefields Field data in mesh nodes to output to file
110 : : //! \param[in] elemsurfs Surface field data in mesh elements to output to file
111 : : //! \param[in] nodesurfs Surface field data in mesh nodes to output to file
112 : : //! \param[in] outsets Unique set of surface side set ids along which to save
113 : : //! solution field variables
114 : : //! \param[in] c Function to continue with after the write
115 : : // *****************************************************************************
116 : : {
117 [ + - ]: 3706 : if (!m_benchmark) {
118 : :
119 : : // Generate filenames for volume and surface field output
120 [ + - ]: 7412 : auto vf = filename( basefilename, meshid, itr, chareid );
121 : :
122 [ + + ]: 3706 : if (meshoutput) {
123 [ + - ]: 976 : if (m_filetype == ctr::FieldFileType::EXODUSII) {
124 : :
125 : : // Write volume mesh and field names
126 [ + - ]: 1952 : ExodusIIMeshWriter ev( vf, ExoWriter::CREATE );
127 : : // Write chare mesh (do not write side sets in parallel)
128 [ + + ]: 976 : if (m_nchare[meshid] == 1) {
129 : :
130 [ + + ]: 94 : if (m_bndCentering == Centering::ELEM)
131 [ + - ]: 52 : ev.writeMesh( inpoel, coord, bface, triinpoel );
132 [ + - ]: 42 : else if (m_bndCentering == Centering::NODE)
133 [ + - ]: 42 : ev.writeMesh( inpoel, coord, bnode );
134 [ - - ][ - - ]: 0 : else Throw( "Centering not handled for writing mesh" );
[ - - ]
135 : :
136 : : } else {
137 [ + - ]: 882 : ev.writeMesh< 4 >( inpoel, coord );
138 : : }
139 [ + - ]: 976 : ev.writeElemVarNames( elemfieldnames );
140 [ - + ][ - - ]: 976 : Assert( nodefieldnames.size() == nodefields.size(), "Size mismatch" );
[ - - ][ - - ]
141 [ + - ]: 976 : ev.writeNodeVarNames( nodefieldnames );
142 : :
143 : : // Write surface meshes and surface variable field names
144 [ + + ]: 988 : for (auto s : outsets) {
145 [ + - ]: 12 : auto sf = filename( basefilename, meshid, itr, chareid, s );
146 [ + - ]: 12 : ExodusIIMeshWriter es( sf, ExoWriter::CREATE );
147 [ + - ]: 12 : auto b = bface.find(s);
148 [ - + ]: 12 : if (b == end(bface)) {
149 : : // If a side set does not exist on a chare, write out a
150 : : // connectivity for a single triangle with its node coordinates of
151 : : // zero. This is so the paraview series reader can load side sets
152 : : // distributed across multiple files. See also
153 : : // https://www.paraview.org/Wiki/Restarted_Simulation_Readers.
154 [ - - ][ - - ]: 0 : es.writeMesh< 3 >( std::vector< std::size_t >{1,2,3},
155 [ - - ][ - - ]: 0 : UnsMesh::Coords{{ {{0,0,0}}, {{0,0,0}}, {{0,0,0}} }} );
[ - - ]
156 [ - - ]: 0 : es.writeElemVarNames( elemsurfnames );
157 [ - - ]: 0 : es.writeNodeVarNames( nodesurfnames );
158 : 0 : continue;
159 : : }
160 : 24 : std::vector< std::size_t > nodes;
161 [ + + ]: 2122 : for (auto f : b->second) {
162 [ + - ]: 2110 : nodes.push_back( triinpoel[f*3+0] );
163 [ + - ]: 2110 : nodes.push_back( triinpoel[f*3+1] );
164 [ + - ]: 2110 : nodes.push_back( triinpoel[f*3+2] );
165 : : }
166 [ + - ]: 24 : auto [inp,gid,lid] = tk::global2local( nodes );
167 [ + - ]: 12 : tk::unique( nodes );
168 : 12 : auto nnode = nodes.size();
169 : 24 : UnsMesh::Coords scoord;
170 [ + - ]: 12 : scoord[0].resize( nnode );
171 [ + - ]: 12 : scoord[1].resize( nnode );
172 [ + - ]: 12 : scoord[2].resize( nnode );
173 : 12 : std::size_t j = 0;
174 [ + + ]: 1477 : for (auto i : nodes) {
175 : 1465 : scoord[0][j] = coord[0][i];
176 : 1465 : scoord[1][j] = coord[1][i];
177 : 1465 : scoord[2][j] = coord[2][i];
178 : 1465 : ++j;
179 : : }
180 [ + - ]: 12 : es.writeMesh< 3 >( inp, scoord );
181 [ + - ]: 12 : es.writeElemVarNames( elemsurfnames );
182 [ + - ]: 12 : es.writeNodeVarNames( nodesurfnames );
183 : : }
184 : :
185 : : }
186 : : }
187 : :
188 [ + - ]: 3706 : if (fieldoutput) {
189 [ + - ]: 3706 : if (m_filetype == ctr::FieldFileType::EXODUSII) {
190 : :
191 : : // Write volume variable fields
192 [ + - ]: 7412 : ExodusIIMeshWriter ev( vf, ExoWriter::OPEN );
193 [ + - ]: 3706 : ev.writeTimeStamp( itf, time );
194 : : // Write volume element variable fields
195 : 3706 : int varid = 0;
196 [ + + ][ + - ]: 18899 : for (const auto& v : elemfields) ev.writeElemScalar( itf, ++varid, v );
197 : : // Write volume node variable fields
198 : 3706 : varid = 0;
199 [ + + ][ + - ]: 15031 : for (const auto& v : nodefields) ev.writeNodeScalar( itf, ++varid, v );
200 : :
201 : : // Write surface node variable fields
202 : 3706 : std::size_t j = 0;
203 : 3706 : std::size_t k = 0;
204 : 3706 : auto nvar = static_cast< int >( nodesurfnames.size() ) ;
205 : 3706 : auto nevar = static_cast< int >( elemsurfnames.size() ) ;
206 [ + + ]: 3758 : for (auto s : outsets) {
207 [ + - ]: 52 : auto sf = filename( basefilename, meshid, itr, chareid, s );
208 [ + - ]: 52 : ExodusIIMeshWriter es( sf, ExoWriter::OPEN );
209 [ + - ]: 52 : es.writeTimeStamp( itf, time );
210 [ + - ][ - + ]: 52 : if (bface.find(s) == end(bface)) {
211 : : // If a side set does not exist on a chare, write out a
212 : : // a node field for a single triangle with zeros. This is so the
213 : : // paraview series reader can load side sets distributed across
214 : : // multiple files. See also
215 : : // https://www.paraview.org/Wiki/Restarted_Simulation_Readers.
216 [ - - ][ - - ]: 0 : for (int i=1; i<=nvar; ++i) es.writeNodeScalar( itf, i, {0,0,0} );
[ - - ]
217 [ - - ][ - - ]: 0 : for (int i=1; i<=nevar; ++i) es.writeElemScalar( itf, i, {0} );
[ - - ]
218 : 0 : continue;
219 : : }
220 [ + + ]: 220 : for (int i=1; i<=nvar; ++i)
221 [ + - ]: 168 : es.writeNodeScalar( itf, i, nodesurfs[j++] );
222 [ + + ]: 364 : for (int i=1; i<=nevar; ++i)
223 [ + - ]: 312 : es.writeElemScalar( itf, i, elemsurfs[k++] );
224 : : }
225 : :
226 : : }
227 : : }
228 : :
229 : : }
230 : :
231 : 3706 : c.send();
232 : 3706 : }
233 : :
234 : : std::string
235 : 3770 : MeshWriter::filename( const std::string& basefilename,
236 : : std::size_t meshid,
237 : : uint64_t itr,
238 : : int chareid,
239 : : int surfid ) const
240 : : // *****************************************************************************
241 : : // Compute filename
242 : : //! \param[in] basefilename String use as the base filename.
243 : : //! \param[in] meshid Mesh Id
244 : : //! \param[in] itr Iteration count since a new mesh. New mesh in this context
245 : : //! means that either the mesh is moved and/or its topology has changed.
246 : : //! \param[in] chareid The chare id the write-to-file request is coming from
247 : : //! \param[in] surfid Surface ID if computing a surface filename
248 : : //! \details We use a file naming convention for large field output data that
249 : : //! allows ParaView to glue multiple files into a single simulation output by
250 : : //! only loading a single file. The base filename is followed by ".e-s.",
251 : : //! which probably stands for Exodus Sequence, followed by 3 integers:
252 : : //! (1) {RS}: counts the number of "restart dumps", but we use this for
253 : : //! counting the number of outputs with a different mesh, e.g., due to
254 : : //! mesh refinement, thus if this first number is new the mesh is new
255 : : //! compared to the previous (first) number afer ".e-s.",
256 : : //! (2) {NP}: total number of partitions (workers, chares), this is more than
257 : : //! the number of PEs with nonzero virtualization (overdecomposition), and
258 : : //! (3) {RANK}: worker (chare) id.
259 : : //! Thus {RANK} does spatial partitioning, while {RS} partitions in time, but
260 : : //! a single {RS} id may contain multiple time steps, which equals to the
261 : : //! number of time steps at which field output is saved without refining the
262 : : //! mesh.
263 : : //! \return Filename computed
264 : : //! \see https://www.paraview.org/Wiki/Restarted_Simulation_Readers
265 : : // *****************************************************************************
266 : : {
267 [ + + ][ + - ]: 7540 : return basefilename + (surfid ? "-surf." + std::to_string(surfid) : "")
[ + - ][ + - ]
[ + - ][ + + ]
[ + + ][ - - ]
[ - - ]
268 [ + + ][ + - ]: 15080 : + (m_nmesh > 1 ? '.' + std::to_string(meshid) : "")
[ + - ][ + - ]
[ + - ][ + + ]
[ + + ][ - - ]
[ - - ]
269 [ + - ]: 7540 : + ".e-s"
270 [ + - ][ + - ]: 15080 : + '.' + std::to_string( itr ) // iteration count with new mesh
[ + - ]
271 [ + - ][ + - ]: 15080 : + '.' + std::to_string( m_nchare[meshid] ) // total number of workers
[ + - ]
272 [ + - ][ + - ]: 15080 : + '.' + std::to_string( chareid ) // new file per worker
273 : : ;
274 : : }
275 : :
276 : : #include "NoWarning/meshwriter.def.h"
|
