46 #ifndef MUELU_COORDINATESTRANSFER_FACTORY_KOKKOS_DEF_HPP 47 #define MUELU_COORDINATESTRANSFER_FACTORY_KOKKOS_DEF_HPP 56 #include "MueLu_Aggregates_kokkos.hpp" 57 #include "MueLu_CoarseMapFactory_kokkos.hpp" 60 #include "MueLu_Utilities_kokkos.hpp" 64 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class DeviceType>
65 RCP<const ParameterList> CoordinatesTransferFactory_kokkos<Scalar,LocalOrdinal,GlobalOrdinal,Kokkos::Compat::KokkosDeviceWrapperNode<DeviceType>>::GetValidParameterList()
const {
66 RCP<ParameterList> validParamList =
rcp(
new ParameterList());
68 validParamList->set< RCP<const FactoryBase> >(
"Coordinates", Teuchos::null,
"Factory for coordinates generation");
69 validParamList->set< RCP<const FactoryBase> >(
"Aggregates", Teuchos::null,
"Factory for coordinates generation");
70 validParamList->set< RCP<const FactoryBase> >(
"CoarseMap", Teuchos::null,
"Generating factory of the coarse map");
71 validParamList->set<
int > (
"write start", -1,
"First level at which coordinates should be written to file");
72 validParamList->set<
int > (
"write end", -1,
"Last level at which coordinates should be written to file");
74 return validParamList;
77 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class DeviceType>
78 void CoordinatesTransferFactory_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Kokkos::Compat::KokkosDeviceWrapperNode<DeviceType>>::DeclareInput(Level& fineLevel, Level& coarseLevel)
const {
79 static bool isAvailableCoords =
false;
82 isAvailableCoords = coarseLevel.IsAvailable(
"Coordinates",
this);
84 if (isAvailableCoords ==
false) {
85 Input(fineLevel,
"Coordinates");
86 Input(fineLevel,
"Aggregates");
87 Input(fineLevel,
"CoarseMap");
91 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class DeviceType>
92 void CoordinatesTransferFactory_kokkos<Scalar, LocalOrdinal, GlobalOrdinal, Kokkos::Compat::KokkosDeviceWrapperNode<DeviceType>>::Build(Level& fineLevel, Level& coarseLevel)
const {
93 FactoryMonitor m(*
this,
"Build", coarseLevel);
97 GetOStream(
Runtime0) <<
"Transferring coordinates" << std::endl;
99 if (coarseLevel.IsAvailable(
"Coordinates",
this)) {
100 GetOStream(
Runtime0) <<
"Reusing coordinates" << std::endl;
104 auto aggregates = Get<RCP<Aggregates_kokkos> >(fineLevel,
"Aggregates");
105 auto fineCoords = Get<RCP<doubleMultiVector> >(fineLevel,
"Coordinates");
106 auto coarseMap = Get<RCP<const Map> > (fineLevel,
"CoarseMap");
112 ArrayView<const GO> elementAList = coarseMap->getNodeElementList();
114 if (rcp_dynamic_cast<const StridedMap>(coarseMap) != Teuchos::null)
115 blkSize = rcp_dynamic_cast<
const StridedMap>(coarseMap)->getFixedBlockSize();
117 GO indexBase = coarseMap->getIndexBase();
118 size_t numElements = elementAList.size() / blkSize;
119 Array<GO> elementList(numElements);
122 for (
LO i = 0; i < Teuchos::as<LO>(numElements); i++)
123 elementList[i] = (elementAList[i*blkSize]-indexBase)/blkSize + indexBase;
125 RCP<const Map> uniqueMap = fineCoords->getMap();
130 RCP<doubleMultiVector> ghostedCoords = fineCoords;
131 if (aggregates->AggregatesCrossProcessors()) {
132 RCP<const Map> nonUniqueMap = aggregates->GetMap();
133 RCP<const Import> importer = ImportFactory::Build(uniqueMap, nonUniqueMap);
140 int myPID = uniqueMap->getComm()->getRank();
141 LO numAggs = aggregates->GetNumAggregates();
142 auto aggSizes = aggregates->ComputeAggregateSizes(
true,
true);
143 auto vertex2AggID = aggregates->GetVertex2AggId()->template getLocalView<DeviceType>();
144 auto procWinner = aggregates->GetProcWinner() ->template getLocalView<DeviceType>();
146 auto fineCoordsView = fineCoords ->template getLocalView<DeviceType>();
147 auto coarseCoordsView = coarseCoords->template getLocalView<DeviceType>();
150 typename AppendTrait<decltype(coarseCoordsView), Kokkos::Atomic>::type coarseCoordsViewAtomic = coarseCoordsView;
151 for (
size_t j = 0; j < fineCoords->getNumVectors(); j++) {
152 Kokkos::parallel_for(
"MueLu:CoordinatesTransferF:Build:coord", vertex2AggID.size(), KOKKOS_LAMBDA(
const LO lnode) {
153 if (procWinner(lnode, 0) == myPID)
154 coarseCoordsViewAtomic(vertex2AggID(lnode, 0),j) += fineCoordsView(lnode,j);
157 Kokkos::parallel_for(
"MueLu:CoordinatesTransferF:Build:agg", numAggs, KOKKOS_LAMBDA(
const LO agg) {
158 coarseCoordsView(agg,j) /= aggSizes[agg];
162 Set<RCP<doubleMultiVector> >(coarseLevel,
"Coordinates", coarseCoords);
164 const ParameterList& pL = GetParameterList();
165 int writeStart = pL.get<
int>(
"write start"), writeEnd = pL.get<
int>(
"write end");
166 if (writeStart == 0 && fineLevel.GetLevelID() == 0 && writeStart <= writeEnd) {
167 std::string fileName =
"coordinates_before_rebalance_level_" +
toString(fineLevel.GetLevelID()) +
".m";
170 if (writeStart <= coarseLevel.GetLevelID() && coarseLevel.GetLevelID() <= writeEnd) {
171 std::string fileName =
"coordinates_before_rebalance_level_" +
toString(coarseLevel.GetLevelID()) +
".m";
178 #endif // MUELU_COORDINATESTRANSFER_FACTORY_KOKKOS_DEF_HPP
std::string toString(const T &what)
Little helper function to convert non-string types to strings.
static void Write(const std::string &fileName, const Xpetra::Map< LocalOrdinal, GlobalOrdinal, Node > &M)
One-liner description of what is happening.
Namespace for MueLu classes and methods.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
static Teuchos::RCP< MultiVector< Scalar, LocalOrdinal, GlobalOrdinal, Node > > Build(const Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > &map, size_t NumVectors, bool zeroOut=true)