From e821d617802ebbfff756844e4d9720342fc78a4e Mon Sep 17 00:00:00 2001 From: "Claudius \"keldu\" Holeksa" Date: Fri, 13 Nov 2026 14:12:49 +0100 Subject: Swarm POC --- examples/cavity_2d_gpu/.nix/derivation.nix | 3 + examples/cavity_2d_gpu/SConstruct | 2 + examples/cavity_2d_gpu/cavity_2d_gpu.cpp | 726 ++++++++++++++++------------- 3 files changed, 408 insertions(+), 323 deletions(-) (limited to 'examples/cavity_2d_gpu') diff --git a/examples/cavity_2d_gpu/.nix/derivation.nix b/examples/cavity_2d_gpu/.nix/derivation.nix index 8724d96..3b210a3 100644 --- a/examples/cavity_2d_gpu/.nix/derivation.nix +++ b/examples/cavity_2d_gpu/.nix/derivation.nix @@ -23,13 +23,16 @@ stdenv.mkDerivation { buildInputs = [ forstio.core + forstio.io forstio.async forstio.codec forstio.codec-unit forstio.remote + forstio.remote-filesystem forstio.codec-json adaptive-cpp kel.lbm.core + kel.lbm.sycl # kel-lbm.sycl ]; diff --git a/examples/cavity_2d_gpu/SConstruct b/examples/cavity_2d_gpu/SConstruct index fe206e1..7882265 100644 --- a/examples/cavity_2d_gpu/SConstruct +++ b/examples/cavity_2d_gpu/SConstruct @@ -59,6 +59,8 @@ env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], ], LIBS=[ 'forstio-core', + 'forstio-async', + 'forstio-io', 'acpp-rt', 'omp' ] diff --git a/examples/cavity_2d_gpu/cavity_2d_gpu.cpp b/examples/cavity_2d_gpu/cavity_2d_gpu.cpp index 55225d1..6b71db3 100644 --- a/examples/cavity_2d_gpu/cavity_2d_gpu.cpp +++ b/examples/cavity_2d_gpu/cavity_2d_gpu.cpp @@ -1,384 +1,464 @@ +#include #include -#include -#include - -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include namespace kel { namespace lbm { + +// Square cavity, as this case has always used. +constexpr uint64_t dim_x = 128ul; +constexpr uint64_t dim_y = 128ul; + +// Lid velocity, tangential along +x on the top row. +constexpr double lid_velocity = 0.1; + namespace sch { -/** - * Basic distribution function - * Base type - * D - * Q - * Scalar factor - * D factor - * Q factor - */ -using T = Float32; -using D2Q9 = Descriptor<2u,9u>; - -template -using DfCell = Cell; - -template -using CellInfo = Cell; - -/** - * Basic type for simulation - */ -template -using CellStruct = CellFieldStruct< - Desc, - Struct< - Member>, "dfs">, - Member>, "dfs_old">, - Member>, "info"> - > ->; +using InfoChunk = Chunk; -template -using MacroStruct = Struct< - Member, "velocity">, - Member ->; +template +using DfChunk = Chunk, 1u, dim_x, dim_y>; -using CavityFieldD2Q9 = CellStruct; -} +template +using ScalarChunk = Chunk, 0u, dim_x, dim_y>; +template +using VectorChunk = Chunk, 0u, dim_x, dim_y>; -/* -template -class df_cell_view; -*/ -/** - * Minor helper for the AA-Pull Pattern, so I can use only one lattice - * - * Am I sure I want to use AA this way? - * Esoteric Twist technically reduces the needed memory access footprint - */ -/* -template -class df_cell_view, Encode> { -public: - using Schema = sch::Cell; -private: - std::array::type>*, QN> view_; -public: - df_cell_view(const std::array::type>*, QN>& view): - view_{view} - {} -}; -*/ -namespace cmpt { -struct MovingWall {}; -} +template +using ChunkStruct = Struct< + Member, + Member, "dfs">, + Member, "dfs_old"> +>; -/** - * Full-Way moving wall Bounce back, something is not right here. - * Technically it should reflect properly. - */ -template -class component { -public: - std::array::type, Desc::D> lid_vel; - -public: - void apply( - kel::lbm::data>& dfs - ){ - using dfi = df_info; - - // Technically use .copy() - /* - auto dfs_cpy = dfs; +template +using VelChunk = Chunk, 0u, dim_x, dim_y>; - for(uint64_t i = 0u; i < Desc::Q; ++i){ - dfs({dfi::opposite_index.at(i)}) = dfs_cpy({i}) - 2.0 * dfi::weights[i] * 1.0 * ( lid_vel[0] * dfi::directions[i][0] + lid_vel[1] * dfi::directions[i][1]) * dfi::inv_cs2; - } - */ - } -}; +template +using RhoChunk = Chunk, 0u, dim_x, dim_y>; -constexpr size_t dim_x = 128; -constexpr size_t dim_y = 128; +template +using MacroStruct = Struct< + Member, "velocity">, + Member, "density"> +>; -template -void set_geometry( - kel::lbm::data>* info_field -){ - using namespace kel::lbm; - using namespace acpp; - kel::lbm::data> meta{{dim_x,dim_y}}; - /** - * Set ghost - */ - iterate_over([&](const kel::lbm::data>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - - info({0u}).set(0u); - - }, {{0u,0u}}, meta); - - /** - * Set wall - */ - iterate_over([&](const kel::lbm::data>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - - info({0u}).set(1u); - - }, {{0u,0u}}, meta, {{1u,1u}}); - - /** - * Set fluid - */ - iterate_over([&](const kel::lbm::data>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - - info({0u}).set(2u); - - }, {{0u,0u}}, meta, {{2u,2u}}); - - /** - * Set top lid - */ - iterate_over([&](const kel::lbm::data>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - info({0u}).set(3u); - - }, {{0u,1u}}, {{meta.at({0u}), 2u}}, {{2u,0u}}); +//template +//using ParticleArray = Array< +// Particle +//>; } -template -void set_initial_conditions( - kel::lbm::data>* info_field, - kel::lbm::data>* dfs_field, - kel::lbm::data>* dfs_old_field +template +kel::lbm::error_or setup_initial_conditions( + kel::lbm::data>& fields, + kel::lbm::data>& macros ){ - using namespace kel::lbm; - using namespace acpp; - - kel::lbm::data rho{1.0}; - kel::lbm::data> vel{{0.0,0.0}}; - auto eq = equilibrium(rho, vel); - kel::lbm::data> meta{{dim_x,dim_y}}; - - /** - * Set distribution - */ - iterate_over([&](const kel::lbm::data>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& dfs = dfs_field[i]; - auto& dfs_old = dfs_old_field[i]; - - for(kel::lbm::data k = 0; k < kel::lbm::data{Desc::Q}; ++k){ - dfs(k) = eq.at(k); - dfs_old(k) = eq.at(k); - } + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u}} + ); + + // The lid: the top row, moving tangentially. Corners stay plain walls. + iterator::apply( + [&](auto& index){ + info_f.at(index).set(5u); + }, + {{0u,dim_y-1u}}, + {{dim_x,dim_y}}, + {{1u,0u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& vel_f = macros.template get<"velocity">(); + + iterator::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& vel = vel_f.at(index); + auto eq = equilibrium(rho,vel); + + df = eq; + }, + {},// 0-index + df_f.get_dims() + ); + + iterator::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& vel = vel_f.at(index); + if(info_f.at(index).get() == 2u){ + vel.at({{0u}}) = 0.0; + } + auto eq = equilibrium(rho,vel); + + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u}} + ); + + iterator::apply( + [&](auto& index){ + kel::lbm::data> middle, ind_vec; + middle.at({{0u}}) = dim_x * 0.25; + middle.at({{1u}}) = dim_y * 0.5; + + ind_vec.at({{0u}}) = index.at({{0u}}).template cast_to(); + ind_vec.at({{1u}}) = index.at({{1u}}).template cast_to(); + + auto dist = middle - ind_vec; + auto dist_2 = kel::lbm::math::dot(dist,dist); + if(dist_2.at({}).get() < dim_y*dim_y*0.01){ + info_f.at(index) = 1u; + } + }, + {},// 0-index + df_f.get_dims() + ); - }, {{0u,0u}}, meta); + return kel::lbm::make_void(); } template -void lbm_step( - kel::lbm::data>* info_r, - kel::lbm::data>* dfs_r, - kel::lbm::data>* dfs_r_old, - bool is_even, - uint64_t time_step, - acpp::sycl::queue& sycl_q +kel::lbm::error_or step( + kel::lbm::data>,encode::Sycl>& fields, + kel::lbm::data>,encode::Sycl>& macros, + kel::lbm::data t_i, + device& dev ){ - using namespace kel::lbm; - using namespace acpp; - - using dfi = df_info; - - //component coll{0.59}; - constexpr kel::lbm::data frequency{1.0 / 0.59}; - //component bb; - component bb_lid; - bb_lid.lid_vel = {0.1, 0.0}; - - // Submit collision kernel - sycl_q.submit([&](sycl::handler& cgh) { - // Accessor for latt with read/write - - cgh.parallel_for( - sycl::range<2>{dim_x, dim_y}, [=](sycl::id<2> idx) { - size_t i = idx[0]; - size_t j = idx[1]; - - size_t acc_id = i * dim_x + j; - - // Read cell info - auto& info = info_r[acc_id]; - - switch (info({0u}).get()) { - case 1u: { - // bb.apply(latt_acc, {i, j}, time_step); + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); + + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + // Need nicer things to handle the flow. I see improvement here + component> collision{0.8}; + component> bb; + component,encode::Sycl> abb; + + kel::lbm::data> rho_b; + rho_b.at({}) = 1.0; + kel::lbm::data> vel_b; + vel_b.at({{0u}}) = 0.015; + + component> equi{rho_b,vel_b}; + + component,encode::Sycl> flow_in{ + [&](){ + uint64_t target_t_i = 32u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.0002 / target_t_i) * t_i.get(); + } + return 1.0002; + }() + }; + component,encode::Sycl> flow_out{1.0}; - auto& dfs_old = is_even ? dfs_r_old[acc_id] : dfs_r[acc_id]; - auto df_cpy = dfs_old.copy(); - for(uint64_t i = 1u; i < Desc::Q; ++i){ - dfs_old({i}) = df_cpy({dfi::opposite_index.at(i)}); - } + h.parallel_for(acpp::sycl::range{dim_x,dim_y}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: break; - } - case 2u: { - // coll.apply(latt_acc, {i, j}, time_step); - auto& dfs_old = is_even ? dfs_r_old[acc_id] : dfs_r[acc_id]; - - kel::lbm::data rho; - kel::lbm::data> vel; - compute_rho_u(dfs_old,rho,vel); - auto eq = equilibrium(rho,vel); - - for(uint64_t i = 0u; i < Desc::Q; ++i){ - dfs_old({i}) = dfs_old({i}) + frequency * (eq.at(i) - dfs_old({i})); - } + case 1u: + bb.apply(fields,index,t_i); break; - } - default: - // Do nothing + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 5u: { + /** + * Moving wall: bounce back plus the tangential momentum the + * lid injects. + * + * f_i = f_opp + 2 w_i rho_w (e_i . u_w) / cs^2 + * + * written from f_opp because that is the direction the + * library's BounceBack writes in. There is no component for a + * y normal moving wall, and the original example had this + * commented out, so it is implemented here. + */ + using dfi = df_info; + const bool is_even = ((t_i.get() % 2u) == 0u); + auto& dfs_f = (is_even) + ? fields.template get<"dfs_old">() + : fields.template get<"dfs">(); + + auto& dfs = dfs_f.at(index); + auto df_cpy = dfs; + + for(uint64_t i = 0u; i < Desc::Q; ++i){ + const auto e_dot_u = + static_cast::type>( + dfi::directions[i][0]) * lid_velocity; + + dfs.at({i}) = df_cpy.at({dfi::opposite_index.at(i)}) + + kel::lbm::data{ + static_cast::type>( + 2.0 * dfi::weights[i] * dfi::inv_cs2 * e_dot_u)}; + } + } break; + default: break; } }); - }); - - // Submit streaming kernel - sycl_q.submit([&](sycl::handler& cgh) { - - cgh.parallel_for( - sycl::range<2>{dim_x - 2, dim_y - 2}, [=](sycl::id<2> idx) { - size_t i = idx[0] + 1; // avoid boundary - size_t j = idx[1] + 1; + }).wait(); - size_t acc_id = i * dim_x + j; + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ - auto& dfs_new = is_even ? dfs_r[acc_id] : dfs_r_old[acc_id]; - auto& info = info_r[acc_id]; - - - if (info({0u}).get() > 0u && info({0u}).get() != 3u) { - for (uint64_t k = 0u; k < Desc::Q; ++k) { - auto dir = dfi::directions[dfi::opposite_index[k]]; - // auto& cell_dir_old = latt_acc({i + dir[0], j + dir[1]}); - // - size_t acc_old_id = (i+dir[0]) * dim_x + (j+dir[1]); - - auto& dfs_old = is_even ? dfs_r_old[acc_old_id] : dfs_r[acc_old_id]; - auto& info_old = info_r[acc_old_id]; - - if (info_old({0}).get() == 3u) { - auto& dfs_old_loc = is_even ? dfs_r_old[acc_id] : dfs_r[acc_id]; - dfs_new({k}) = dfs_old_loc({dfi::opposite_index.at(k)}) - 2.0 * dfi::inv_cs2 * dfi::weights.at(k) * 1.0 * (bb_lid.lid_vel[0] * dir[0] + bb_lid.lid_vel[1] * dir[1]); - } else { - dfs_new({k}) = dfs_old({k}); - } - } - } - }); - }); + return kel::lbm::make_void(); } } } -int main(){ +template +kel::lbm::error_or lbm_main(int argc, char** argv){ using namespace kel::lbm; - using namespace acpp; - - kel::lbm::data> meta{{dim_x, dim_y}}; - constexpr size_t dim_size = dim_x * dim_y; - converter conv{ - {0.1}, - {0.1} - }; + /* + auto eo_lbm_args = setup_lbm_env,sch::Tuple<>>>(argc,argv); + if(eo_lbm_args.is_error()){ + return std::move(eo_lbm_args.get_error()); + } + auto& lbm_args = eo_lbm_args.get_value(); + */ - // Lid driven cavity: the lid velocity and the cavity height are the - // characteristic scales. - print_lbm_meta(conv, {1e-3}, {0.1}, {dim_y}); + using dfi = df_info; auto eo_lbm_dir = output_directory(); if(eo_lbm_dir.is_error()){ - return -1; + return std::move(eo_lbm_dir.get_error()); } auto& lbm_dir = eo_lbm_dir.get_value(); - auto out_dir = lbm_dir / "cavity_gpu_2d"; - sycl::queue sycl_q{sycl::default_selector_v, sycl::property::queue::in_order{}}; + auto out_dir = lbm_dir / "cavity_2d_gpu"; - constexpr size_t num_cells = dim_x * dim_y; - - // Allocate USM shared memory for your main data - auto* info = sycl::malloc_shared>>(num_cells, sycl_q); - auto* dfs = sycl::malloc_shared>>(num_cells, sycl_q); - auto* dfs_old = sycl::malloc_shared>>(num_cells, sycl_q); + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error("Could not create output directory"); + } + } - /** - * Set meta information describing what this cell is - */ - set_geometry(info); + converter conv { + // delta_x + {1.0}, + // delta_t + {1.0} + }; + + print_lbm_meta(conv,{0.1},{1e-4},{0.4 * dim_y}); + + // kel::lbm::data> meta{{dim_x,dim_y}}; + auto lbm_data_ptr = kel::lbm::heap>>(); + auto lbm_macro_data_ptr = kel::lbm::heap>>(); + + std::cout<<"Estimated Bytes: "<,sch::MacroStruct>().get()<( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; - /** - * - */ - set_initial_conditions(info,dfs,dfs_old); + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); - /** - * Timeloop - */ - uint64_t lattice_steps = 512000u; - bool even_step = true; + device dev; - uint64_t print_every = 256u; - uint64_t file_no = 0u; + auto& sycl_q = dev.get_handle(); - kel::lbm::data,sch::D2Q9::D>> macros{meta}; + sycl_q.wait(); + { + auto eov = setup_initial_conditions(*lbm_data_ptr,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"initial_state",0u,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } - std::cout<<"Start"<, encode::Sycl> lbm_sycl_data{sycl_q}; + kel::lbm::data, encode::Sycl> lbm_sycl_macro_data{sycl_q}; + sycl_q.wait(); - auto start = std::chrono::steady_clock::now(); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } sycl_q.wait(); - for(uint64_t i = 0u; i < lattice_steps; ++i){ - lbm_step(info, dfs, dfs_old, even_step, i, sycl_q); - if(i % 1024u == 0u){ - sycl_q.wait(); - iterate_over([&](const kel::lbm::data>& index){ - size_t j = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto dfs_field = even_step ? dfs_old : dfs; - - auto& rho = macros.at(index).template get<"pressure">(); - auto& vel = macros.at(index).template get<"velocity">(); - compute_rho_u(dfs_field[j],rho,vel); - }, {{0u,0u}}, meta); - std::string vtk_f_name{"tmp/cavity_2d_gpu_"}; - vtk_f_name += std::to_string(i) + ".vtk"; - write_vtk_file(vtk_f_name, macros); + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + kel::lbm::data time_steps{16u*4096ul}; + + auto& info_f = lsd_view.template get<"info">(); + + for(kel::lbm::data i{0u}; i < time_steps and krun; ++i){ + // BC + Collision + { + auto eov = step(lsd_view,lsdm_view,i,dev); + if(eov.is_error()){ + return eov; + } } + sycl_q.wait(); + /* + if(i.get() % 128u == 0u){ + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + */ + /* + if(i.get() % 32u == 0u){ + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_csv_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + */ + // Stream + sycl_q.submit([&](acpp::sycl::handler& h){ + component> stream; + + h.parallel_for(acpp::sycl::range{dim_x,dim_y}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(lsd_view,index,i); + } + }); + }).wait(); + wait.poll(); + if(print_status){ + std::cout<<"Status: "<().get() * 100 / time_steps.get())<<"%"<(argc, argv); + if(eov.is_error()){ + auto& err = eov.get_error(); + std::cerr<<"[Error] "< 0u){ + std::cerr<<" - "<