Branch data     Line data    Source code

       1                 :            : // *****************************************************************************
       2                 :            : /*!
       3                 :            :   \file      src/LoadBalance/LinearMap.cpp
       4                 :            :   \copyright 2012-2015 J. Bakosi,
       5                 :            :              2016-2018 Los Alamos National Security, LLC.,
       6                 :            :              2019-2021 Triad National Security, LLC.
       7                 :            :              All rights reserved. See the LICENSE file for details.
       8                 :            :   \brief     Advanced Charm++ array creation with a map in a linear fashion
       9                 :            :   \details   Advanced Charm++ array creation refers to various ways arrays can
      10                 :            :      be created with the Charm++ runtime system. See
      11                 :            :      http://charm.cs.illinois.edu/manuals/html/charm++/manual.html, Sec.
      12                 :            :      Advanced Array Creation. This class does a simple linear distribution.
      13                 :            : 
      14                 :            :      Note that to help with performance, it is not advised to do heavy
      15                 :            :      computations in the overridden member functions, procNum() and
      16                 :            :      populateInitial(), since they can be potentially called many times. See
      17                 :            :      also this note on the inner workings of the Charm++ runtime system
      18                 :            :      regarding map objects and array element creation from long-time Charm++
      19                 :            :      developer, Eric Bohm, also at:
      20                 :            :      http://lists.cs.uiuc.edu/pipermail/charm/2015-May/002047.html
      21                 :            : 
      22                 :            :      _Procnum will be called to construct your chare, the first time a message
      23                 :            :      is sent to it from a node, and each time subsequent sends do not find a
      24                 :            :      precached location record for it. The latter event can occur when many
      25                 :            :      sends have pushed it out of cache, or after migration._
      26                 :            : 
      27                 :            :      _The potential global memory footprint of location management caching is
      28                 :            :      proportional to the total number of objects multiplied by the number of
      29                 :            :      nodes. Therefore, the runtime system keeps a finite number on each node.
      30                 :            :      At the limit, procnum could be called for nearly every message send,
      31                 :            :      therefore procnum should be designed to be inexpensive._
      32                 :            : 
      33                 :            :      The heavy portion of array element placement should therefore be done in
      34                 :            :      the constructor.
      35                 :            : */
      36                 :            : // *****************************************************************************
      37                 :            : 
      38                 :            : #include <string>
      39                 :            : 
      40                 :            : #include "NoWarning/charm.hpp"
      41                 :            : #include "LinearMap.hpp"
      42                 :            : 
      43                 :            : using tk::LinearMap;
      44                 :            : 
      45                 :            : int
      46                 :         90 : LinearMap::procNum( int, const CkArrayIndex& idx )
      47                 :            : // *****************************************************************************
      48                 :            : //  Return the home processor number for the array element for linear
      49                 :            : //  distribution
      50                 :            : //! \param[in] idx Charm++ array index object containing the array element index
      51                 :            : //!   to assign a PE to
      52                 :            : //! \return PE assigned
      53                 :            : // *****************************************************************************
      54                 :            : {
      55                 :         90 :   int elem = *idx.data();       // array element we assign PE for
      56         [ -  + ]:         90 :   auto pe = elem / m_chunksize;
      57                 :          0 :   if (pe >= CkNumPes()) pe = CkNumPes()-1;
      58                 :            : 
      59                 :            :   Assert( pe < CkNumPes(), "Assigned PE (" + std::to_string(pe) +
      60                 :            :           ") larger than NumPEs (" + std::to_string(CkNumPes()) + ")" );
      61                 :            : 
      62                 :         90 :   return pe;
      63                 :            : }
      64                 :            : 
      65                 :            : void
      66                 :         72 : LinearMap::populateInitial( int, CkArrayOptions& opt, void *msg, CkArrMgr *mgr )
      67                 :            : // *****************************************************************************
      68                 :            : // Create initial set of array elements based on linear distribution
      69                 :            : //! \param[in] opt Charm++ array options object containing the number of initial
      70                 :            : //!   array elements to be created
      71                 :            : //! \param[in] msg Charm++ messsage to use for array element creation
      72                 :            : //! \param[in] mgr Array manager to use
      73                 :            : // *****************************************************************************
      74                 :            : {
      75                 :         72 :   int nelem = *opt.getNumInitial().data(); // number of array elements requested
      76         [ +  - ]:         72 :   if (nelem == 0) return;                  // no initial elements requested
      77                 :            : 
      78                 :         72 :   auto lower = CkMyPe() * m_chunksize;
      79         [ +  + ]:         72 :   auto upper = lower + m_chunksize;
      80                 :         72 :   auto remainder = nelem % CkNumPes();
      81 [ +  + ][ +  + ]:         72 :   if (remainder && CkMyPe() == CkNumPes()-1) upper += remainder;
      82                 :            : 
      83         [ +  + ]:       3312 :   for (int e=0; e<nelem; ++e)
      84         [ +  + ]:       3240 :     if (e >= lower && e < upper)
      85                 :         90 :       mgr->insertInitial( CkArrayIndex(e), CkCopyMsg(&msg) );
      86                 :            : 
      87                 :         72 :   mgr->doneInserting();
      88                 :         72 :   CkFreeMsg( msg );
      89                 :            : }
      90                 :            : 
      91                 :            : #include "NoWarning/linearmap.def.h"