From e821d617802ebbfff756844e4d9720342fc78a4e Mon Sep 17 00:00:00 2001 From: "Claudius \"keldu\" Holeksa" Date: Fri, 13 Nov 2026 14:12:49 +0100 Subject: Swarm POC --- examples/poiseulle_3d/.nix/derivation.nix | 4 +- examples/poiseulle_3d/SConstruct | 2 + examples/poiseulle_3d/poiseulle_3d.cpp | 713 +++++++++++++++--------------- 3 files changed, 363 insertions(+), 356 deletions(-) (limited to 'examples/poiseulle_3d') diff --git a/examples/poiseulle_3d/.nix/derivation.nix b/examples/poiseulle_3d/.nix/derivation.nix index b090add..02fcb8c 100644 --- a/examples/poiseulle_3d/.nix/derivation.nix +++ b/examples/poiseulle_3d/.nix/derivation.nix @@ -21,14 +21,16 @@ stdenv.mkDerivation { buildInputs = [ forstio.core + forstio.remote-filesystem + forstio.io forstio.async forstio.codec forstio.codec-unit forstio.codec-json forstio.remote - forstio.remote-sycl adaptive-cpp kel.lbm.core + kel.lbm.sycl ]; preferLocalBuild = true; diff --git a/examples/poiseulle_3d/SConstruct b/examples/poiseulle_3d/SConstruct index fe206e1..7882265 100644 --- a/examples/poiseulle_3d/SConstruct +++ b/examples/poiseulle_3d/SConstruct @@ -59,6 +59,8 @@ env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], ], LIBS=[ 'forstio-core', + 'forstio-async', + 'forstio-io', 'acpp-rt', 'omp' ] diff --git a/examples/poiseulle_3d/poiseulle_3d.cpp b/examples/poiseulle_3d/poiseulle_3d.cpp index 141dc5f..34d1fbc 100644 --- a/examples/poiseulle_3d/poiseulle_3d.cpp +++ b/examples/poiseulle_3d/poiseulle_3d.cpp @@ -1,428 +1,431 @@ +#include #include +#include -#include - -namespace saw { -/* -template -class data, encode::Sycl> final { -public: - using Schema = kel::lbm::sch::CellField; - using MetaSchema = typename meta_schema::MetaSchema; -private: - data, encode::Sycl> inner_; -public: - data() = delete; - data(const data& inner_meta__, acpp::sycl::queue& q): - inner_{inner_meta__,q} - {} - - const data meta() const { - return inner_.dims(); - } - - template - data get_dim_size() const { - static_assert(i < Desc::D, "Not enough dimensions"); - return inner_.template get_dim_size(); - } - - const data& operator()(const data, Encode>& index)const{ - return inner_.at(index); - } - - data& operator()(const data, Encode>& index){ - return inner_.at(index); - } - - const data& at(const data, Encode>& index)const{ - return inner_.at(index); - } - - data& at(const data, Encode>& index){ - return inner_.at(index); - } - - ref, encode::Sycl >> inner_data() { - return {inner_}; - } - - data internal_size() const { - return inner_.internal_size(); - } - - data* internal_data() { - return inner_.internal_data(); - } -}; -*/ -/* -template -class data>, encode::Sycl> final { -public: - using Schema = schema::Ref>; -private: - data>, encode::Sycl>> inner_ref_; -public: - data() = delete; - data(ref,encode::Sycl>> inner_ref__): - inner_ref_{inner_ref__().inner_data()} - {} - - const data meta() const { - return inner_ref_().dims(); - } - - const data& operator()(const data, Encode>& index)const{ - return inner_ref_().at(index); - } - - data& operator()(const data, Encode>& index){ - return inner_ref_().at(index); - } - - const data& at(const data, Encode>& index)const{ - return inner_ref_().at(index); - } - - data& at(const data, Encode>& index){ - return inner_ref_().at(index); - } - - data internal_size() const { - return inner_ref_().internal_size(); - } - - data* internal_data() { - return inner_ref_().internal_data(); - } -}; - - -template -class data>, encode::Sycl> final { -public: - using Schema = schema::Ref>; -private: - data>, encode::Sycl>> inner_ref_; -public: - data() = delete; - data(ref,encode::Sycl>> inner_ref__): - inner_ref_{inner_ref__().inner_data()} - {} - - const data meta() const { - return inner_ref_().dims(); - } - - const data& operator()(const data, Encode>& index)const{ - return inner_ref_().at(index); - } - - data& operator()(const data, Encode>& index){ - return inner_ref_().at(index); - } - - const data& at(const data, Encode>& index)const{ - return inner_ref_().at(index); - } - - data& at(const data, Encode>& index){ - return inner_ref_().at(index); - } - - data internal_size() const { - return inner_ref_().internal_size(); - } - - data* internal_data() { - return inner_ref_().internal_data(); - } -}; -*/ - -} +#include +#include +#include +#include namespace kel { namespace lbm { -namespace sch { -using T = Float32; -using Desc = Descriptor<3u,27u>; -using DfCell = Cell; +// The channel this case has always used. +constexpr uint64_t dim_x = 2048ul; +constexpr uint64_t dim_y = 128ul; +constexpr uint64_t dim_z = 128ul; +constexpr uint64_t particle_size = 4ul; -using CellInfo = Cell; +namespace sch { -using CellStruct = Struct< - Member, - Member, - //Member, "force">, - Member +using LbmArgs = Args< + Struct< + Member, + Member + >, + Tuple<> >; -} -using SyclKelAllocator = acpp::sycl::usm_allocator, acpp::sycl::usm::alloc::shared>; -uint64_t to_flat_index(const kel::lbm::data>& index, const kel::lbm::data>& meta){ - auto flat = index.at({2u}) + meta.at({2u})*(index.at({1u}) + index.at({0u})*meta.at({1u})); - return flat.get(); -} - -template -kel::lbm::error_or set_geometry(std::vector,SyclKelAllocator>& latt, const kel::lbm::data>& meta){ - using namespace kel::lbm; - - /** - * Set ghost - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); - - info({0u}).set(0u); - }, {{0u,0u}}, meta); - - kel::lbm::data> distance; - - /** - * Set wall - */ - iterator<1u>::apply([&](const kel::lbm::data>& index){ - distance.at(index.at({0u})) = {1u}; - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); - - info({0u}).set(2u); - }, {}, meta, distance); - - /** - * Set fluid - */ - iterator<1u>::apply([&](const kel::lbm::data>& index){ - distance.at(index.at({0u})) = {2u}; - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); +using InfoChunk = Chunk; - info({0u}).set(1u); +template +using DfChunk = Chunk, 1u, dim_x, dim_y, dim_z>; - }, {}, meta, distance); +template +using ScalarChunk = Chunk, 0u, dim_x, dim_y, dim_z>; - /** - * Set inflow - */ +template +using VectorChunk = Chunk, 0u, dim_x, dim_y, dim_z>; - kel::lbm::data> shifted_point; - iterator<1u>::apply([&](const kel::lbm::data>& index){ - distance.at(index.at({0u})) = (index.at({0u}).get() == 0u)?1u:2u; - shifted_point.at(index.at({0u})) = (index.at({0u}).get() == 0u)?3u:meta.at({index.at({0u})}); - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); - - info({0u}).set(3u); - - }, {}, shifted_point, distance); - - /** - * Set outflow - */ - iterator<1u>::apply([&](const kel::lbm::data>& index){ - distance.at(index.at({0u})) = (index.at({0u}).get() == 0u)?1u:2u; - shifted_point.at(index.at({0u})) = (index.at({0u}).get() == 0u)?(meta.at({0u})-3u):0u; - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); +template +using ChunkStruct = Struct< + Member, + Member, "dfs">, + Member, "dfs_old">, + Member, "particle_N">, + Member, "particle_D"> +>; - info({0u}).set(4u); +template +using MomChunk = Chunk, 0u, dim_x, dim_y, dim_z>; - }, shifted_point, meta, distance); +template +using RhoChunk = Chunk, 0u, dim_x, dim_y, dim_z>; - return kel::lbm::make_void(); +template +using MacroStruct = Struct< + Member, "velocity">, + Member, "density">, + Member, "porosity"> +>; } -template -kel::lbm::error_or set_initial_conditions(std::vector,SyclKelAllocator>& latt, const kel::lbm::data>& meta){ - - kel::lbm::data rho{1.0}; - kel::lbm::data> vel{}; - vel.at({0u}) = {0.1f}; - auto eq = equilibrium(rho, vel); - - /** - * Set distribution - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& dfs = cell.template get<"dfs">(); - auto& dfs_old = cell.template get<"dfs_old">(); - - for(kel::lbm::data k = 0; k < kel::lbm::data{sch::Desc::Q}; ++k){ - dfs(k) = eq.at(k); - dfs_old(k) = eq.at(k); - } - }, {}, meta); +template +kel::lbm::error_or setup_initial_conditions( + converter& conv, + kel::lbm::data>& fields, + kel::lbm::data>& macros, + kel::lbm::data, particle_size>>& particles +){ + // Unit conversion is not exercised in this reduced case. + (void) conv; + + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u,1u}} + ); + + // Inflow + iterator::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u,0u}}, + {{1u,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + + // Outflow + iterator::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u,0u}}, + {{dim_x,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& mom_f = macros.template get<"velocity">(); + auto& por_f = macros.template get<"porosity">(); + + iterator::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + por_f.at(index).at({}) = {1}; + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + auto eq = equilibrium(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims() + ); + + iterator::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + mom.at({{0u}}) = 0.01; + auto eq = equilibrium(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u,1u}} + ); + + // No particles in this case -- the array is carried only so the schema + // matches poiseulle_3d_gpu. + (void) particles; + return kel::lbm::make_void(); } -template -kel::lbm::error_or lbm_step( - std::vector,SyclKelAllocator>& latt, - const kel::lbm::data>& meta, - kel::lbm::data time_step, - acpp::sycl::queue& sycl_q +template +kel::lbm::error_or step( + converter& conv, + kel::lbm::data>,encode::Sycl>& fields, + kel::lbm::data>,encode::Sycl>& macros, + kel::lbm::data t_i, + device& dev ){ - using dfi = df_info; - bool even_step = ((time_step.get() % 2u) == 0u); - - component coll{0.5384}; - component bb; - - /** - * Collision - */ - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); - - switch(info({0u}).get()){ - case 1u: { - bb.apply(&latt[0u], meta, index, time_step); - break; + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); + + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + // Need nicer things to handle the flow. I see improvement here + component> collision{conv.template kinematic_viscosity_si_to_tau({0.1})}; + // component> collision{0.6}; + component> bb; + + component,encode::Sycl> flow_in{ + [&](){ + uint64_t target_t_i = 256u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.001 / target_t_i) * t_i.get(); + } + return 1.001; + }() + }; + component,encode::Sycl> flow_out{1.0}; + + h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); } - case 2u: { - coll.apply(&latt[0u], meta, index, time_step); + + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: break; - } - case 3u: { - //inlet.apply(latt, index, time_step); + case 1u: + bb.apply(fields,index,t_i); break; - } - case 4u: { - //outlet.apply(latt, index, time_step); + case 2u: + collision.apply(fields,macros,index,t_i); break; - } - default: + case 3u: + flow_in.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); break; - } + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: + break; + } + }); + }).wait(); - }, {}, meta); - - // Stream - iterator::apply([&](const kel::lbm::data>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& df_new = even_step ? cell.template get<"dfs">() : cell.template get<"dfs_old">(); - auto& info_new = cell.template get<"info">(); - - if(info_new({0u}).get() > 0u){ - for(uint64_t k = 0u; k < sch::Desc::Q; ++k){ - auto dir = dfi::directions[dfi::opposite_index[k]]; - auto index_dir = index; - for(uint64_t i = 0; i < sch::Desc::D; ++i){ - index_dir.at({i}) = index_dir.at({i}) + dir[i]; - } + q.submit([&](acpp::sycl::handler& h){ + component> stream; - auto findex_dir = to_flat_index(index_dir,meta); - auto& cell_dir_old = latt.at(findex_dir); + h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(fields,index,t_i); + } + }); + }).wait(); - auto& df_old = even_step ? cell_dir_old.template get<"dfs_old">() : cell_dir_old.template get<"dfs">(); + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ - df_new({k}) = df_old({k}); - } - } - }, {}, meta); return kel::lbm::make_void(); } +} +} -kel::lbm::error_or real_main(int argc, char** argv){ - using dfi = df_info; +template +kel::lbm::error_or lbm_main(int argc, char** argv){ + using namespace kel::lbm; + + kel::lbm::data args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("poiseulle_3d"); + auto& steps_arg = args_n.template get<"time_steps">(); + steps_arg.set(64u); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); + } + const auto& lbm_env = eo_lbm_env.get_value(); + + auto& out_dir = lbm_env.name_dir; + + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error("Could not create output directory"); + } + } - converter conv { - {0.1f}, - {0.1f} + converter conv { + // delta_x + {1.0}, + // delta_t + {1.0} }; - print_lbm_meta(conv, {1e-3f}); - acpp::sycl::queue sycl_q; + print_lbm_meta(conv,{0.1},{0.01},{1024.0}); - /** - * Set the meta and initialize the lattice - */ - kel::lbm::data> meta{{2048u,128u,128u}}; + auto lbm_data_ptr = kel::lbm::heap>>(); + auto lbm_macro_data_ptr = kel::lbm::heap>>(); + auto lbm_particle_data_ptr = kel::lbm::heap, particle_size>>>(); + + std::cout<<"Estimated Bytes: "<,sch::MacroStruct>().get()<,SyclKelAllocator> lattice{(meta.at({0u})*meta.at({1u})*meta.at({2u})).get(), sycl_shared_alloc}; + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; - /** - * Set the geometry - */ + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); + + device dev; + + auto& sycl_q = dev.get_handle(); + + sycl_q.wait(); { - auto eov = set_geometry(lattice, meta); + auto eov = setup_initial_conditions(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); if(eov.is_error()){ return eov; } } - /** - * Set initial conditions in the simulations - */ + kel::lbm::data, encode::Sycl> lbm_sycl_data{sycl_q}; + kel::lbm::data, encode::Sycl> lbm_sycl_macro_data{sycl_q}; + kel::lbm::data, particle_size>, encode::Sycl> lbm_sycl_particle_data{sycl_q}; + sycl_q.wait(); + + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } { - auto eov = set_initial_conditions(lattice, meta); + auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); if(eov.is_error()){ return eov; } } + sycl_q.wait(); + kel::lbm::data time_steps = args.template get<"args">().template get<"time_steps">(); + + for(kel::lbm::data i{0u}; i < time_steps and krun; ++i){ + { + { + auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + /*{ + std::string file_name = "p_"; + file_name += std::to_string(i.get()); + file_name += ".json"; + auto eov = saw::easy::encode_and_write_file,particle_size>,saw::encode::Json>(out_dir, *lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + }*/ + } + { + auto eov = step(conv,lsd_view,lsdm_view,i,dev); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } - for(kel::lbm::data ts{0u}; ts < kel::lbm::data{1024u*32u}; ++ts){ - auto eov = lbm_step(lattice,meta,ts,sycl_q); + wait.poll(); + if(print_status){ + std::cout<<"Status: "<().get() * 100 / time_steps.get())<<"%"<::apply( + [&](auto& index){ + std::cout<().at(index).template cast_to().get()<<"\n"; + }, + {{0u,0u}}, + {{dim_x, dim_y}} + ); + */ + + sycl_q.wait(); return kel::lbm::make_void(); } -} // lbm -} // kel +using FloatT = kel::lbm::sch::Float32; -/** - * main, but I don't like the error handling - */ int main(int argc, char** argv){ - auto eov = kel::lbm::real_main(argc, argv); + auto eov = lbm_main(argc, argv); if(eov.is_error()){ auto& err = eov.get_error(); + std::cerr<<"[Error] "< 0u){ std::cerr<<" - "<