Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/Main/RNGTestPrint.hpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief RNGTest-specific pretty printer functionality
9 : : \details RNGTest-specific pretty printer functionality.
10 : : */
11 : : // *****************************************************************************
12 : : #ifndef RNGTestPrint_h
13 : : #define RNGTestPrint_h
14 : :
15 : : #include "Types.hpp"
16 : : #include "RNGPrint.hpp"
17 : : #include "RNGTest/InputDeck/InputDeck.hpp"
18 : : #include "Flip_map.hpp"
19 : :
20 : : namespace rngtest {
21 : :
22 : : extern ctr::InputDeck g_inputdeck_defaults;
23 : : extern ctr::InputDeck g_inputdeck;
24 : :
25 : : //! RNGTestPrint : tk::RNGPrint
26 : : class RNGTestPrint : public tk::RNGPrint {
27 : :
28 : : public:
29 : : //! Constructor
30 : : //! \param[in] screen Screen output filename
31 : : //! \param[in,out] str Verbose stream
32 : : //! \param[in] mode Open mode for screen output file, see
33 : : //! http://en.cppreference.com/w/cpp/io/ios_base/openmode
34 : : //! \param[in,out] qstr Quiet stream
35 : : //! \see tk::RNGPrint::RNGPrint and tk::Print::Print
36 : 500 : explicit RNGTestPrint( const std::string& screen,
37 : : std::ostream& str = std::clog,
38 : : std::ios_base::openmode mode = std::ios_base::out,
39 : 500 : std::ostream& qstr = std::cout ) :
40 : 500 : RNGPrint( screen, str, mode, qstr ) {}
41 : :
42 : : //! Bring vanilla overloads from base into scope in case local overloads fail
43 : : using Print::section;
44 : : using Print::item;
45 : :
46 : : //! Print section only if differs from default
47 : : template< class Option, typename... tags >
48 : : void Section() const {
49 : : if (g_inputdeck.get< tags... >() !=
50 : : g_inputdeck_defaults.get< tags... >()) {
51 : : Option opt;
52 : : auto& group = opt.group();
53 : : auto& value = opt.name( g_inputdeck.get< tags... >() );
54 : : m_stream << m_section_title_value_fmt % m_section_indent
55 : : % m_section_bullet
56 : : % group
57 : : % value;
58 : : m_stream << m_section_underline_fmt
59 : : % m_section_indent
60 : : % std::string( m_section_indent.size() + 3 +
61 : : group.size() + value.size(), '-' );
62 : : }
63 : : }
64 : :
65 : : //! Print control option: 'group : option' only if differs from its default
66 : : template< class Option, typename... tags>
67 : : void Item() const {
68 : : if (g_inputdeck.get<tags...>() != g_inputdeck_defaults.get<tags...>()) {
69 : : Option opt;
70 : : m_stream << m_item_name_value_fmt % m_item_indent
71 : : % opt.group()
72 : : % opt.name(g_inputdeck.get<tags...>());
73 : : }
74 : : }
75 : :
76 : : //! Print battery only if differs from default
77 : : //! \param[in] ntest Number tests in battery
78 : : //! \param[in] nstat Number statistics in battery
79 : 4 : void battery( std::size_t ntest, std::size_t nstat ) const {
80 : 4 : if (g_inputdeck.get< tag::selected, tag::battery >() !=
81 [ + - ]: 4 : g_inputdeck_defaults.get< tag::selected, tag::battery >() ) {
82 [ + - ]: 8 : ctr::Battery b;
83 : 4 : auto& group = b.group();
84 [ + - ]: 4 : auto& value = b.name( g_inputdeck.get< tag::selected, tag::battery >() );
85 [ + - ]: 4 : std::stringstream ss;
86 [ + - ][ + - ]: 4 : ss << value << " (" << ntest << " tests, " << nstat << " stats)";
[ + - ][ + - ]
[ + - ][ + - ]
87 : 12 : m_stream << m_section_title_value_fmt % m_section_indent
88 [ + - ][ + - ]: 4 : % m_section_bullet
89 [ + - ]: 4 : % group
90 [ + - ][ + - ]: 8 : % ss.str();
[ + - ]
91 : 4 : m_stream << m_section_underline_fmt
92 [ + - ]: 4 : % m_section_indent
93 : 12 : % std::string( m_section_indent.size() + 3 +
94 [ + - ][ + - ]: 12 : group.size() + ss.str().size(), '-' );
[ + - ][ + - ]
95 : : }
96 : 4 : }
97 : :
98 : : //! Print statistical test name(s)
99 : : //! \param[in] testnames Names of tests
100 : 224 : void names( const std::vector< std::string >& testnames ) const {
101 [ + + ]: 560 : for (const auto& n : testnames)
102 [ + - ][ + - ]: 336 : m_stream << m_list_item_fmt % m_item_indent % n;
[ + - ]
103 : 224 : }
104 : :
105 : : //! Print statistical tests header (with legend)
106 : : //! \param[in] t String to use as title
107 : : //! \param[in] npval Number of p-values from tests
108 : : //! \param[in] ntest Number of tests
109 : 4 : void statshead( const std::string& t, std::size_t npval, std::size_t ntest )
110 : : const {
111 [ + - ]: 4 : std::stringstream ss;
112 [ + - ][ + - ]: 4 : ss << t << " (" << npval << " stats from " << ntest << " tests)";
[ + - ][ + - ]
[ + - ][ + - ]
113 : 12 : m_stream << m_section_title_fmt % m_section_indent
114 [ + - ][ + - ]: 4 : % m_section_bullet
115 [ + - ][ + - ]: 8 : % ss.str();
[ + - ]
116 : 4 : m_stream << m_section_underline_fmt
117 [ + - ]: 4 : % m_section_indent
118 [ + - ][ + - ]: 8 : % std::string( m_section_indent.size() + 2 + ss.str().size(),
119 [ + - ][ + - ]: 4 : '-');
120 [ + - ][ + - ]: 12 : raw( m_item_indent + "Legend: [done/total/failed] Test, RNG : p-value\n" +
121 [ + - ][ + - ]: 8 : m_item_indent + "(eps means a value < 1.0e-300)\n" +
122 [ + - ][ + - ]: 12 : m_item_indent + "(eps1 means a value < 1.0e-15)\n\n" );
123 : :
124 : 4 : }
125 : :
126 : : //! \brief Print one-liner info for test
127 : : //! \details Columns: [done/total/failed]
128 : : //! - done: number of tests completed so far (note that a test may produce
129 : : //! more than one statistics and thus p-values)
130 : : //! - total: total number of tests: number of tests in the suite times the
131 : : //! number of RNGs tested (note that a test may produce more than one
132 : : //! statistics and thus p-values)
133 : : //! - failed: number of failed tests by a given RNG so far
134 : : //! name of the statistical test
135 : : //! name of RNG
136 : : //! result of test: "pass" or "fail, p-value = ..."
137 : : //! \param[in] ncomplete Number of completed tests
138 : : //! \param[in] ntest Total number of tests
139 : : //! \param[in] nfail Number of failed tests for RNG
140 : : //! \param[in] status Vector of vector of string with the following assumed
141 : : //! structure:
142 : : //! - status[0]: vector of name(s) of the test(s),
143 : : //! length: number of p-values
144 : : //! - status[1]: vector of p-value strings: "pass" or "fail, p-value = ...",
145 : : //! length: number of p-values
146 : : //! - status[2]: vector of length 1: RNG name used to run the test
147 : 264 : void test( std::size_t ncomplete,
148 : : std::size_t ntest,
149 : : std::map< std::string, std::size_t >& nfail,
150 : : const std::vector< std::vector< std::string > >& status ) const
151 : : {
152 [ + - ]: 264 : const auto& numfail = nfail.find( status[2][0] );
153 [ - + ][ - - ]: 264 : Assert( numfail != nfail.end(), "Cannot find RNG" );
[ - - ][ - - ]
154 : :
155 : : // Lambda to count number of failed tests
156 : 818 : auto nfailed = [ &numfail, &status ]() {
157 [ + + ][ + + ]: 1404 : for (const auto& p : status[1]) if (p.size() > 4) ++numfail->second;
158 : 396 : return numfail->second;
159 : 264 : };
160 : :
161 : : // Construct and echo info-line for all statistics resulted from test
162 [ + + ]: 660 : for (std::size_t t=0; t<status[0].size(); ++t) {
163 [ + - ]: 396 : std::stringstream ss;
164 [ + - ][ + - ]: 396 : ss << "[" << ncomplete << "/" << ntest << "/" << nfailed()
[ + - ][ + - ]
[ + - ][ + - ]
165 [ + - ][ + - ]: 396 : << "] " << status[0][t];
166 [ + + ][ + - ]: 396 : if (t==0) ss << ", " << status[2][0];
[ + - ]
167 : 396 : (status[1][t] == "pass" ? m_stream : m_qstream) <<
168 [ + + ][ + - ]: 396 : m_item_widename_value_fmt % m_item_indent % ss.str() % status[1][t];
[ + - ][ + - ]
[ + - ][ + - ]
169 : : }
170 : 264 : }
171 : :
172 : : //! \brief Print failed statistical test names, RNGs, and p-values
173 : : //! \details Requirements on the template argument, class Failed: must have
174 : : //! public fields test, rng, and pval.
175 : : //! \param[in] t String to use as title
176 : : //! \param[in] npval Number of p-values from tests
177 : : //! \param[in] nfail Number of failed tests for RNG
178 : : template< class Failed >
179 : 2 : void failed( const std::string& t,
180 : : std::size_t npval,
181 : : const std::vector< Failed >& nfail ) const
182 : : {
183 [ + - ]: 4 : std::stringstream ss;
184 [ + - ][ + - ]: 2 : ss << t << " (" << nfail.size() << "/" << npval << ")";
[ + - ][ + - ]
[ + - ][ + - ]
185 [ + - ][ + - ]: 2 : section( ss.str() );
186 [ + - ][ + - ]: 2 : raw( m_item_indent + "The following tests gave p-values outside "
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ][ + - ]
[ + - ]
187 : : "[0.001, 0.999]\n" +
188 : 2 : m_item_indent + "List groupped by RNG, in the order given in the "
189 : : "input file\n" +
190 : 2 : m_item_indent + "Legend: Test, RNG : p-value\n" +
191 : 2 : m_item_indent + "(eps means a value < 1.0e-300)\n" +
192 [ + - ]: 2 : m_item_indent + "(eps1 means a value < 1.0e-15)\n\n" );
193 : 4 : std::string oldname;
194 [ + + ]: 20 : for (const auto& f : nfail) {
195 [ + - ]: 36 : std::string newname( f.rng );
196 [ + + ][ + - ]: 18 : std::string rngname( newname == oldname ? "" : (", " + newname) );
[ + - ][ + + ]
[ - - ]
197 [ + - ]: 18 : oldname = newname;
198 : 36 : m_stream << m_item_widename_value_fmt
199 [ + - ]: 18 : % m_item_indent
200 [ + - ]: 36 : % (f.test + rngname)
201 [ + - ][ + - ]: 36 : % f.pval;
[ + - ]
202 : : }
203 : 2 : }
204 : :
205 : : //! Print RNGs and their measured run times
206 : : //! \param[in] name Section name
207 : : //! \param[in] costnote A note on how to interpret the costs
208 : : //! \param[in] c Costs for RNGs
209 : 2 : void cost( const std::string& name,
210 : : const std::string& costnote,
211 : : std::map< std::string, tk::real > c ) const
212 : : {
213 [ - + ][ - - ]: 2 : Assert( !c.empty(), "Empty map passed to cost()" );
[ - - ][ - - ]
214 [ + - ]: 4 : std::multimap< tk::real, std::string > times = tk::flip_map( c );
215 [ + - ]: 2 : section< tk::QUIET >( name );
216 [ + - ][ + - ]: 2 : raw< tk::QUIET >( m_item_indent + costnote + "\n\n" );
[ + - ]
217 : 2 : tk::real fastest = times.begin()->first;
218 [ + + ]: 8 : for (const auto& t : times) {
219 [ + - ]: 6 : std::stringstream ss;
220 [ + - ][ + - ]: 6 : ss << t.first << " (" << std::setprecision(3) << t.first/fastest
[ + - ][ + - ]
221 [ + - ]: 6 : << "x)";
222 [ + - ][ + - ]: 6 : item< tk::QUIET >( t.second, ss.str() );
223 : : }
224 : 2 : }
225 : :
226 : : //! Print RNGs and their number of failed tests
227 : : //! \param[in] name Section name
228 : : //! \param[in] ranknote A note on how to interpret ranks
229 : : //! \param[in] f Ranks for RNGs
230 : 2 : void rank( const std::string& name,
231 : : const std::string& ranknote,
232 : : std::map< std::string, std::size_t > f ) const
233 : : {
234 [ - + ][ - - ]: 2 : Assert( !f.empty(), "Empty map passed to rank()" );
[ - - ][ - - ]
235 [ + - ]: 4 : std::multimap< std::size_t, std::string > nfail = tk::flip_map( f );
236 [ + - ]: 2 : section< tk::QUIET >( name );
237 [ + - ][ + - ]: 2 : raw< tk::QUIET >( m_item_indent + ranknote + "\n\n" );
[ + - ]
238 [ + + ][ + - ]: 8 : for (const auto& t : nfail) item< tk::QUIET >( t.second, t.first );
239 : 2 : }
240 : : };
241 : :
242 : : } // rngtest::
243 : :
244 : : #endif // RNGTestPrint_h
|
