Branch data Line data Source code
1 : : // *****************************************************************************
2 : : /*!
3 : : \file src/LoadBalance/ZoltanInterOp.cpp
4 : : \copyright 2012-2015 J. Bakosi,
5 : : 2016-2018 Los Alamos National Security, LLC.,
6 : : 2019-2021 Triad National Security, LLC.
7 : : All rights reserved. See the LICENSE file for details.
8 : : \brief Interoperation with the Zoltan library
9 : : \details Interoperation with the Zoltan library, used for static mesh graph
10 : : partitioning.
11 : : */
12 : : // *****************************************************************************
13 : :
14 : : #include <stddef.h>
15 : : #include <string>
16 : :
17 : : #include "NoWarning/Zoltan2_PartitioningProblem.hpp"
18 : :
19 : : #include "ZoltanInterOp.hpp"
20 : :
21 : : namespace tk {
22 : : namespace zoltan {
23 : :
24 : : //! GeometricMeshElemAdapter : Zoltan2::MeshAdapter
25 : : //! \details GeometricMeshElemAdapter specializes those virtual member functions
26 : : //! of Zoltan2::MeshAdapter that are required for mesh-element-based
27 : : //! geometric partitioning with Zoltan2
28 : : template< typename ZoltanTypes >
29 : 666 : class GeometricMeshElemAdapter : public Zoltan2::MeshAdapter< ZoltanTypes > {
30 : :
31 : : private:
32 : : using MeshEntityType = Zoltan2::MeshEntityType;
33 : : using EntityTopologyType = Zoltan2::EntityTopologyType;
34 : :
35 : : public:
36 : : using gno_t = typename Zoltan2::InputTraits< ZoltanTypes >::gno_t;
37 : : using scalar_t = typename Zoltan2::InputTraits< ZoltanTypes >::scalar_t;
38 : : using base_adapter_t = Zoltan2::MeshAdapter< ZoltanTypes >;
39 : :
40 : : //! Constructor
41 : : //! \param[in] nelem Number of elements in mesh graph on this rank
42 : : //! \param[in] centroid Mesh element coordinates (centroids)
43 : : //! \param[in] elemid Mesh element global IDs
44 [ + - ]: 666 : GeometricMeshElemAdapter(
45 : : std::size_t nelem,
46 : : const std::array< std::vector< tk::real >, 3 >& centroid,
47 : : const std::vector< long >& elemid )
48 : : : m_nelem( nelem ),
49 : : m_topology( EntityTopologyType::TETRAHEDRON ),
50 : : m_centroid( centroid ),
51 : 666 : m_elemid( elemid )
52 : : {}
53 : :
54 : : //! Returns the number of mesh entities on this rank
55 : : //! \return Number of mesh elements on this rank
56 : : // cppcheck-suppress unusedFunction
57 : 1867 : std::size_t getLocalNumOf( MeshEntityType ) const override
58 : 1867 : { return m_nelem; }
59 : :
60 : : //! Provide a pointer to this rank's identifiers
61 : : //! \param[in,out] Ids Pointer to the list of global element Ids on this
62 : : //! rank
63 : : // cppcheck-suppress unusedFunction
64 : 904 : void getIDsViewOf( MeshEntityType, const gno_t*& Ids) const override
65 : 904 : { Ids = m_elemid.data(); }
66 : :
67 : : //! Provide a pointer to the entity topology types
68 : : //! \param Types Pointer to the list of entity topology types on this rank
69 : : // cppcheck-suppress unusedFunction
70 : 0 : void getTopologyViewOf( MeshEntityType,
71 : : const EntityTopologyType*& Types ) const override
72 : 0 : { Types = &m_topology; }
73 : :
74 : : //! Return dimensionality of the mesh
75 : : //! \return Number of mesh dimension
76 : : // cppcheck-suppress unusedFunction
77 : 666 : int getDimension() const override { return 3; }
78 : :
79 : : //! Provide a pointer to one dimension of mesh element coordinates
80 : : //! \param[in] coords Pointer to a list of coordinate values for the
81 : : //! dimension
82 : : //! \param[in,out] stride Describes the layout of the coordinate values in
83 : : //! the coords list. If stride is one, then the ith coordinate value is
84 : : //! coords[i], but if stride is two, then the ith coordinate value is
85 : : //! coords[2*i]
86 : : //! \param dim Value from 0 to one less than getEntityCoordinateDimension()
87 : : //! specifying which dimension is being provided in the coords list
88 : : // cppcheck-suppress unusedFunction
89 : 1998 : void getCoordinatesViewOf( MeshEntityType,
90 : : const scalar_t*& coords,
91 : : int &stride,
92 : : int dim ) const override
93 : : {
94 : 1998 : coords = m_centroid[ static_cast<std::size_t>(dim) ].data();
95 : 1998 : stride = 1;
96 : 1998 : }
97 : :
98 : : private:
99 : : //! Number of elements on this rank
100 : : const std::size_t m_nelem;
101 : : //! Mesh element topology types
102 : : const EntityTopologyType m_topology;
103 : : //! Mesh element coordinates (centroids)
104 : : const std::array< std::vector< tk::real >, 3 >& m_centroid;
105 : : //! Global mesh element ids
106 : : const std::vector< long >& m_elemid;
107 : : };
108 : :
109 : : std::vector< std::size_t >
110 : 666 : geomPartMesh( tk::ctr::PartitioningAlgorithmType algorithm,
111 : : const std::array< std::vector< tk::real >, 3 >& centroid,
112 : : const std::vector< long >& elemid,
113 : : int npart )
114 : : // *****************************************************************************
115 : : // Partition mesh using Zoltan2 with a geometric partitioner, such as RCB, RIB
116 : : //! \param[in] algorithm Partitioning algorithm type
117 : : //! \param[in] centroid Mesh element coordinates
118 : : //! \param[in] elemid Global mesh element ids
119 : : //! \param[in] npart Number of desired graph partitions
120 : : //! \return Array of chare ownership IDs mapping graph points to concurrent
121 : : //! async chares
122 : : //! \details This function uses Zoltan to partition the mesh graph in parallel.
123 : : //! It assumes that the mesh graph is distributed among all the MPI ranks.
124 : : // *****************************************************************************
125 : : {
126 : : // Set Zoltan parameters
127 [ + - ][ + - ]: 1998 : Teuchos::ParameterList params( "Zoltan parameters" );
[ + - ]
128 [ + - ][ + - ]: 2664 : params.set( "algorithm", tk::ctr::PartitioningAlgorithm().param(algorithm) );
[ + - ][ + - ]
[ - + ][ + - ]
[ - - ]
129 [ + - ][ + - ]: 1998 : params.set( "num_global_parts", std::to_string(npart) );
[ + - ][ - + ]
[ - + ][ + - ]
[ - - ]
130 [ + - ][ + - ]: 1332 : params.set( "objects_to_partition", "mesh_elements" );
[ + - ][ - + ]
131 : :
132 : : // Define types for Zoltan2
133 : : // * 1st argument, 'scalar': the data type for element values, weights and
134 : : // coordinates
135 : : // * 2nd argument, 'lno' (local number): the integral data type used by
136 : : // the application, i.e, quinoa, and by Zoltan2 for local indices and local
137 : : // counts
138 : : // * 3rd argument 'gno' (global number): is the integral data type used by
139 : : // the application, i.e., quinoa, and Zoltan2 to represent global
140 : : // identifiers and global counts
141 : : // See also
142 : : // external/src/trilinos/packages/zoltan2/src/input/Zoltan2_InputTraits.hpp
143 : : using ZoltanTypes = Zoltan2::BasicUserTypes< tk::real, long, long >;
144 : :
145 : : // Create mesh adapter for Zoltan for mesh element partitioning
146 : : using InciterZoltanAdapter = GeometricMeshElemAdapter< ZoltanTypes >;
147 [ + - ]: 666 : InciterZoltanAdapter adapter( elemid.size(), centroid, elemid );
148 : :
149 : : // Create Zoltan2 partitioning problem using our mesh input adapter
150 : : Zoltan2::PartitioningProblem< InciterZoltanAdapter >
151 [ + - ]: 1332 : partitioner( &adapter, ¶ms );
152 : :
153 : : // Perform partitioning using Zoltan
154 [ + - ]: 666 : partitioner.solve();
155 : :
156 : : // Copy over array of chare IDs corresponding to the ownership of elements
157 : : // in our chunk of the mesh graph, i.e., the coloring or chare ids for the
158 : : // mesh elements we operated on
159 : : auto partlist = partitioner.getSolution().getPartListView();
160 [ + - ]: 666 : std::vector< std::size_t > chare( elemid.size() );
161 [ + + ]: 2315342 : for (std::size_t p=0; p<elemid.size(); ++p )
162 : 2314676 : chare[p] = static_cast< std::size_t >( partlist[p] );
163 : :
164 : 666 : return chare;
165 : : }
166 : :
167 : : } // zoltan::
168 : : } // tk::
|
