Skip to content

Commit 04512d0

Browse files
Sichao25Sichao25
committed
Merge branch 'develop' into yus/ci
2 parents 640ed82 + 6256e61 commit 04512d0

30 files changed

Lines changed: 811 additions & 934 deletions

‎.github/workflows/clang-tidy.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ jobs:
105105
with:
106106
repo-name: 'meshFields-openmpi'
107107
repo-path: 'SCOREC/meshFields'
108-
repo-ref: 'b1482bbba288df210784b2345eae08e34faabdc4'
108+
repo-ref: 'b9984520c20f73955a3b4e25e4907780b91d430f'
109109
cache: true
110110
options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
111111
-DCMAKE_C_COMPILER=`which mpicc`
@@ -165,7 +165,7 @@ jobs:
165165
with:
166166
repo-name: 'redev-openmpi'
167167
repo-path: 'SCOREC/redev'
168-
repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
168+
repo-ref: '1e4c876f786b420f060695ba2c1079447e975b69'
169169
cache: true
170170
options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
171171
-DMPIEXEC_EXECUTABLE=`which mpirun`

‎.github/workflows/cmake-test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ jobs:
132132
with:
133133
repo-name: 'meshFields'
134134
repo-path: 'SCOREC/meshFields'
135-
repo-ref: 'b1482bbba288df210784b2345eae08e34faabdc4'
135+
repo-ref: 'b9984520c20f73955a3b4e25e4907780b91d430f'
136136
cache: true
137137
cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
138138
options: '-DCMAKE_CXX_COMPILER=`which mpicxx`
@@ -165,7 +165,7 @@ jobs:
165165
with:
166166
repo-name: 'redev'
167167
repo-path: 'SCOREC/redev'
168-
repo-ref: 'ac09848a5f9b89493e8b679c9080b9efe5538376'
168+
repo-ref: '1e4c876f786b420f060695ba2c1079447e975b69'
169169
cache: true
170170
cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
171171
options: '-DCMAKE_CXX_COMPILER=`which mpicxx`

‎src/pcms/coupler/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ set(PCMS_COUPLER_HEADERS
77
field_exchange_planner.h
88
partition.h
99
overlap_mask.h
10+
global_communicator.h
1011
)
1112

1213

‎src/pcms/coupler/coupler.hpp‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,48 @@
1111
#include "pcms/utility/assert.h"
1212
#include "pcms/utility/common.h"
1313
#include "pcms/utility/profile.h"
14+
#include "pcms/coupler/global_communicator.h"
1415
#include <memory>
1516

1617
namespace pcms
1718
{
19+
template <typename T>
20+
class GlobalDataInterface
21+
{
22+
public:
23+
GlobalDataInterface(std::string name,
24+
Rank1View<T, pcms::HostMemorySpace> data,
25+
MPI_Comm mpi_comm, redev::Channel& channel)
26+
: data_(data),
27+
variable_name_(name),
28+
mpi_comm_(mpi_comm),
29+
comm_(name, mpi_comm_, channel)
30+
{
31+
PCMS_FUNCTION_TIMER;
32+
}
33+
34+
void Send(redev::Mode mode = redev::Mode::Synchronous)
35+
{
36+
PCMS_FUNCTION_TIMER;
37+
comm_.Send(data_, variable_name_, mode);
38+
}
39+
40+
void Receive(redev::Mode mode = redev::Mode::Synchronous)
41+
{
42+
PCMS_FUNCTION_TIMER;
43+
comm_.Receive(data_, variable_name_, mode);
44+
}
45+
46+
private:
47+
Rank1View<T, pcms::HostMemorySpace> data_;
48+
std::string variable_name_;
49+
MPI_Comm mpi_comm_;
50+
GlobalCommunicator<T> comm_;
51+
};
52+
using GlobalDataVariant =
53+
std::variant<GlobalDataInterface<int8_t>, GlobalDataInterface<int32_t>,
54+
GlobalDataInterface<int64_t>, GlobalDataInterface<float>,
55+
GlobalDataInterface<double>>;
1856

1957
class Application;
2058

@@ -31,6 +69,21 @@ class FieldHandle
3169
void Receive(redev::Mode mode = redev::Mode::Synchronous) const;
3270
[[nodiscard]] Field<T>& GetField() const;
3371

72+
private:
73+
Application* app_;
74+
std::string name_;
75+
};
76+
template <typename T>
77+
class DataHandle
78+
{
79+
public:
80+
DataHandle(Application* app, std::string name)
81+
: app_(app), name_(std::move(name))
82+
{
83+
}
84+
void Send(redev::Mode mode = redev::Mode::Synchronous) const;
85+
void Receive(redev::Mode mode = redev::Mode::Synchronous) const;
86+
3487
[[nodiscard]] const std::string& GetName() const noexcept { return name_; }
3588

3689
private:
@@ -99,6 +152,16 @@ class Application
99152
// Register a transferable field: like AddField but retains the function space
100153
// (via the stored Function) and returns a FunctionHandle usable in transfers.
101154
template <typename T>
155+
FieldHandle<T> AddField(std::string name, Field<T>&& field,
156+
std::unique_ptr<FieldSerializer<T>> serializer,
157+
bool participates = true);
158+
// Registers a reference to user-owned data. The Application does not take
159+
// ownership; it uses the registered reference for subsequent send and receive
160+
// operations.
161+
template <typename T>
162+
DataHandle<T> AddData(std::string name, Rank1View<T, HostMemorySpace> data,
163+
MPI_Comm mpi_comm);
164+
template <typename T>
102165
FunctionHandle<T> AddFunction(Function<T>&& function,
103166
bool participates = true);
104167

@@ -127,6 +190,25 @@ class Application
127190
[mode](auto& field_communicator) { field_communicator->Receive(); },
128191
detail::find_or_error(name, field_communicators_));
129192
};
193+
void SendData(const std::string& name,
194+
redev::Mode mode = redev::Mode::Synchronous)
195+
{
196+
PCMS_FUNCTION_TIMER;
197+
PCMS_ALWAYS_ASSERT(InSendPhase());
198+
199+
std::visit([mode](auto& data_interface) { data_interface.Send(mode); },
200+
detail::find_or_error(name, global_data_interfaces_));
201+
}
202+
203+
void ReceiveData(const std::string& name,
204+
redev::Mode mode = redev::Mode::Synchronous)
205+
{
206+
PCMS_FUNCTION_TIMER;
207+
PCMS_ALWAYS_ASSERT(InReceivePhase());
208+
209+
std::visit([mode](auto& data_interface) { data_interface.Receive(mode); },
210+
detail::find_or_error(name, global_data_interfaces_));
211+
}
130212
[[nodiscard]] bool InSendPhase() const noexcept
131213
{
132214
PCMS_FUNCTION_TIMER;
@@ -205,8 +287,39 @@ class Application
205287
std::map<std::string, std::unique_ptr<FieldLayoutCommunicator>>
206288
field_layout_communicators_;
207289
std::map<std::string, std::unique_ptr<OverlapMask>> layout_overlap_masks_;
290+
std::map<std::string, GlobalDataVariant> global_data_interfaces_;
208291
};
209292

293+
template <typename T>
294+
DataHandle<T> Application::AddData(std::string name,
295+
Rank1View<T, HostMemorySpace> data,
296+
MPI_Comm mpi_comm)
297+
{
298+
PCMS_FUNCTION_TIMER;
299+
auto [it, inserted] = global_data_interfaces_.try_emplace(
300+
name, std::in_place_type<GlobalDataInterface<T>>, name, data, mpi_comm,
301+
channel_);
302+
if (!inserted) {
303+
throw pcms_error("Global data interface with this name already exists");
304+
}
305+
return DataHandle<T>{this, std::move(name)};
306+
}
307+
308+
template <typename T>
309+
void DataHandle<T>::Send(redev::Mode mode) const
310+
{
311+
PCMS_ALWAYS_ASSERT(app_ != nullptr);
312+
313+
app_->SendData(name_, mode);
314+
}
315+
template <typename T>
316+
void DataHandle<T>::Receive(redev::Mode mode) const
317+
{
318+
PCMS_ALWAYS_ASSERT(app_ != nullptr);
319+
320+
app_->ReceiveData(name_, mode);
321+
}
322+
210323
class Coupler
211324
{
212325
private:

‎src/pcms/coupler/global_communicator.h‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#ifndef PCMS_GLOBAL_COMMUNICATOR_H
2+
#define PCMS_GLOBAL_COMMUNICATOR_H
3+
#include <redev.h>
4+
#include <pcms/utility/profile.h>
5+
#include <pcms/utility/assert.h>
6+
namespace pcms
7+
{
8+
using redev::Mode;
9+
template <typename T>
10+
struct GlobalCommunicator
11+
{
12+
using value_type = T;
13+
14+
public:
15+
GlobalCommunicator(std::string name, MPI_Comm mpi_comm,
16+
redev::Channel& channel)
17+
: mpi_comm(mpi_comm), channel_(channel), name_(std::move(name))
18+
{
19+
PCMS_FUNCTION_TIMER;
20+
comm_ = channel_.CreateComm<T>(name_, mpi_comm, redev::CommType::Global);
21+
}
22+
GlobalCommunicator(const GlobalCommunicator&) = delete;
23+
GlobalCommunicator& operator=(const GlobalCommunicator&) = delete;
24+
GlobalCommunicator(GlobalCommunicator&&) = default;
25+
GlobalCommunicator& operator=(GlobalCommunicator&&) = default;
26+
27+
void Send(Rank1View<T, HostMemorySpace> msg, std::string VarName,
28+
Mode mode = Mode::Synchronous)
29+
{
30+
PCMS_FUNCTION_TIMER;
31+
PCMS_ALWAYS_ASSERT(channel_.InSendCommunicationPhase());
32+
auto msg_size = static_cast<std::size_t>(msg.extent(0));
33+
comm_.SetCommParams(VarName, msg_size);
34+
comm_.Send(msg.data_handle(), mode);
35+
}
36+
void Receive(Rank1View<T, HostMemorySpace> destination, std::string VarName,
37+
Mode mode = Mode::Synchronous)
38+
{
39+
PCMS_FUNCTION_TIMER;
40+
PCMS_ALWAYS_ASSERT(channel_.InReceiveCommunicationPhase());
41+
auto msg_size = static_cast<std::size_t>(destination.extent(0));
42+
comm_.SetCommParams(VarName, msg_size);
43+
comm_.Recv(destination.data_handle(), msg_size, mode);
44+
}
45+
46+
private:
47+
MPI_Comm mpi_comm;
48+
redev::Channel& channel_;
49+
std::string name_;
50+
redev::BidirectionalComm<T> comm_;
51+
};
52+
} // namespace pcms
53+
#endif // PCMS_GLOBAL_COMMUNICATOR_H

‎src/pcms/field/evaluator/mesh_fields_backend.h‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include <Kokkos_Core.hpp>
88
#include <MeshField.hpp>
9+
#include <MeshField_Config.hpp>
910
#include <memory>
1011

1112
#include "pcms/field/layout/mesh_fields.h"
@@ -53,7 +54,7 @@ class MeshFieldBackendImpl : public MeshFieldBackend<T>
5354
{
5455
auto self = const_cast<MeshFieldBackendImpl<T, Dim, Order>*>(this);
5556
return self->mesh_field_.triangleLocalPointEval(localCoords, offsets,
56-
shape_field_);
57+
shape_field_.field);
5758
}
5859

5960
void SetData(Rank1View<const T, DeviceMemorySpace> data, size_t num_nodes,
@@ -65,7 +66,7 @@ class MeshFieldBackendImpl : public MeshFieldBackend<T>
6566
mesh_.nents(dim), KOKKOS_CLASS_LAMBDA(size_t ent) {
6667
for (size_t n = 0; n < num_nodes; ++n) {
6768
for (size_t c = 0; c < num_components; ++c) {
68-
shape_field_(ent, n, c, topo) =
69+
shape_field_.field(ent, n, c, topo) =
6970
data[ent * stride + n * num_components + c];
7071
}
7172
}
@@ -82,7 +83,7 @@ class MeshFieldBackendImpl : public MeshFieldBackend<T>
8283
for (size_t n = 0; n < num_nodes; ++n) {
8384
for (size_t c = 0; c < num_components; ++c) {
8485
data[ent * stride + n * num_components + c] =
85-
shape_field_(ent, n, c, topo);
86+
shape_field_.field(ent, n, c, topo);
8687
}
8788
}
8889
});
@@ -91,9 +92,8 @@ class MeshFieldBackendImpl : public MeshFieldBackend<T>
9192
private:
9293
Omega_h::Mesh& mesh_;
9394
MeshField::OmegahMeshField<DefaultExecutionSpace, Dim> mesh_field_;
94-
using ShapeField =
95-
decltype(mesh_field_.template CreateLagrangeField<T, Order, 1>());
96-
ShapeField shape_field_;
95+
using FWC = decltype(mesh_field_.template CreateLagrangeField<T, Order, 1>());
96+
FWC shape_field_; // FWC = FieldWithController; keeps ctrlr alive
9797
};
9898

9999
// ---------------------------------------------------------------------------
@@ -240,7 +240,7 @@ struct FillCoordinatesAndIndicesFunctor
240240
const auto owner_idx = owning_elem_ids_(i);
241241
LO count = Kokkos::atomic_sub_fetch(&elem_counts_(owner_idx), 1);
242242
LO index = offsets_(owner_idx) + count - 1;
243-
for (int j = 0; j < (dim_ + 1); ++j) {
243+
for (int j = 0; j < dim_; ++j) {
244244
coordinates_(index, j) = coord[j];
245245
}
246246
indices_(index) = i;
@@ -404,8 +404,9 @@ struct FillCoordinatesDeviceFunctor
404404
global_coords_(orig_idx, 1)};
405405
const auto local =
406406
Omega_h::barycentric_from_global<2, 2>(point, vertex_coords);
407-
for (int j = 0; j < (dim_ + 1); ++j)
407+
for (int j = 0; j < dim_; ++j) {
408408
coordinates_(index, j) = local[j];
409+
}
409410
indices_(index) = orig_idx;
410411
}
411412
};
@@ -495,7 +496,7 @@ struct MeshFieldsAdapter2LocalizationHint
495496

496497
offsets_ = Kokkos::View<LO*, HostMemorySpace>("offsets", mesh.nelems() + 1);
497498
coordinates_ = Kokkos::View<Real**, HostMemorySpace>(
498-
"coordinates", num_valid_, mesh.dim() + 1);
499+
"coordinates", num_valid_, mesh.dim());
499500
indices_ = Kokkos::View<LO*, HostMemorySpace>("indices", num_valid_);
500501

501502
if (num_missing_ > 0) {
@@ -537,8 +538,9 @@ struct MeshFieldsAdapter2LocalizationHint
537538
global_coords(orig_idx, 1)};
538539
const auto local =
539540
Omega_h::barycentric_from_global<2, 2>(point, vertex_coords);
540-
for (int j = 0; j < (mesh.dim() + 1); ++j)
541+
for (int j = 0; j < mesh.dim(); ++j) {
541542
coordinates_(index, j) = local[j];
543+
}
542544
indices_(index) = static_cast<LO>(orig_idx);
543545
}
544546

@@ -632,7 +634,7 @@ struct MeshFieldsAdapter2LocalizationHint
632634

633635
// Step 6: Fill coordinates and indices on device
634636
coordinates_d_ =
635-
Kokkos::View<Real**>("coordinates_d", num_valid_, mesh.dim() + 1);
637+
Kokkos::View<Real**>("coordinates_d", num_valid_, mesh.dim());
636638
indices_d_ = Kokkos::View<LO*>("indices_d", num_valid_);
637639
const auto tris2verts = mesh.ask_elem_verts();
638640
const auto mesh_coords = mesh.coords();
@@ -654,7 +656,7 @@ struct MeshFieldsAdapter2LocalizationHint
654656
// Create host mirrors for compatibility (lazy copy - only if needed)
655657
offsets_ = Kokkos::create_mirror_view(offsets_d_);
656658
coordinates_ = Kokkos::View<Real**, HostMemorySpace>(
657-
"coordinates_", num_valid_, mesh.dim() + 1);
659+
"coordinates_", num_valid_, mesh.dim());
658660
DeepCopyMismatchLayouts(coordinates_, coordinates_d_);
659661
indices_ = Kokkos::create_mirror_view(indices_d_);
660662
if (num_missing_ > 0) {
@@ -681,4 +683,4 @@ struct MeshFieldsAdapter2LocalizationHint
681683

682684
} // namespace pcms
683685

684-
#endif // PCMS_ADAPTER_MESHFIELDS_MESH_FIELDS_BACKEND_H
686+
#endif // PCMS_ADAPTER_MESHFIELDS_MESH_FIELDS_BACKEND_H

‎src/pcms/transfer/mass_matrix_integrator.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#include <KokkosController.hpp>
55
#include <MeshField.hpp>
6+
#include <MeshField_Config.hpp>
67
#include <MeshField_Element.hpp>
78
#include <MeshField_Fail.hpp>
89
#include <MeshField_For.hpp>
@@ -38,7 +39,7 @@ class MassMatrixIntegrator : public MeshField::Integrator
3839
// std::cerr << " Number points per Elem : " << numPtsPerElem << "\n";
3940
assert(numPtsPerElem >= 1);
4041
const size_t ptDim = p.extent(1);
41-
assert(ptDim == fe.MeshEntDim + 1);
42+
assert(ptDim == fe.MeshEntDim);
4243
// Copy values needed in the kernel to avoid capturing host references
4344
// (mesh and fe are host objects and cannot be dereferenced on the device)
4445
const auto numElems = mesh.nelems();
@@ -51,8 +52,7 @@ class MassMatrixIntegrator : public MeshField::Integrator
5152
const auto last = first + numPtsPerElem;
5253
for (auto pt = first; pt < last; pt++) {
5354
// FIXME better way to fill? pass kokkos::subview to getValues?
54-
Kokkos::Array<MeshField::Real, FieldElement::MeshEntDim + 1>
55-
localCoord;
55+
Kokkos::Array<MeshField::Real, FieldElement::MeshEntDim> localCoord;
5656
for (auto i = 0; i < localCoord.size(); i++) {
5757
localCoord[i] = p(pt, i);
5858
}

0 commit comments

Comments
 (0)