diff options
| author | Claudius "keldu" Holeksa <mail@keldu.de> | 2026-11-11 01:16:44 +0100 |
|---|---|---|
| committer | Claudius "keldu" Holeksa <mail@keldu.de> | 2026-11-11 01:16:44 +0100 |
| commit | 0c0087ef9482634273ec1a23d3a915754f94926d (patch) | |
| tree | 972a8e48c1fae67af23b982d7a9e1a506fc8705f /examples | |
| parent | 2e63c4a31f946a8bafa92c49c1a0a116c0eb6dae (diff) | |
| download | libs-lbm-0c0087ef9482634273ec1a23d3a915754f94926d.tar.gz | |
Snapshot. Migration to Lbm Unit
Diffstat (limited to 'examples')
65 files changed, 2536 insertions, 271 deletions
diff --git a/examples/cavity_2d/.nix/derivation.nix b/examples/cavity_2d/.nix/derivation.nix index 9ae46b5..d0660e9 100644 --- a/examples/cavity_2d/.nix/derivation.nix +++ b/examples/cavity_2d/.nix/derivation.nix @@ -5,7 +5,7 @@ , forstio , pname , version -, kel-lbm +, kel }: stdenv.mkDerivation { @@ -24,7 +24,7 @@ stdenv.mkDerivation { forstio.codec forstio.codec-unit forstio.codec-json - kel-lbm.core + kel.lbm.core ]; preferLocalBuild = true; diff --git a/examples/cavity_2d/cavity_2d.cpp b/examples/cavity_2d/cavity_2d.cpp index 177bf05..ea63745 100644 --- a/examples/cavity_2d/cavity_2d.cpp +++ b/examples/cavity_2d/cavity_2d.cpp @@ -39,10 +39,18 @@ using CellStruct = Struct< Member<CellInfo<Desc>, "info"> >; -template<typename T, uint64_t D> +/** + * Macroscopic fields, one chunk per quantity. + * + * This used to be an Array of per cell structs. The VTK writer takes a Struct + * whose members are all Chunks of the same extent, and compute_rho_u wants + * Scalar/Vector rather than a bare primitive and a FixedArray, so the layout + * moved from array-of-struct to struct-of-arrays. + */ +template<typename T, uint64_t D, uint64_t DimX, uint64_t DimY> using MacroStruct = Struct< - Member<FixedArray<T,D>, "momentum">, - Member<T, "pressure"> + Member<Chunk<Vector<T,D>, 0u, DimX, DimY>, "velocity">, + Member<Chunk<Scalar<T>, 0u, DimX, DimY>, "density"> >; using CavityFieldD2Q9 = CellField<D2Q9, CellStruct<D2Q9>>; @@ -267,7 +275,9 @@ int main(){ {0.1} }; - print_lbm_meta<sch::T, sch::D2Q9>(conv, {1e-3}); + // Lid driven cavity: the lid velocity and the cavity height are the + // characteristic scales. + print_lbm_meta<sch::T, sch::D2Q9>(conv, {1e-3}, {0.1}, {dim_y}); //auto& df_field = lattices.at(0).template get<"dfs">(); //for(uint64_t i = 0; i < df_field.get_dim_size<0u>(); ++i){ @@ -308,7 +318,8 @@ int main(){ uint64_t print_every = 256u; uint64_t file_no = 0u; - kel::lbm::data<sch::Array<sch::MacroStruct<sch::T,sch::D2Q9::D>,sch::D2Q9::D>> macros{dim}; + // Chunks carry their extent in the type, so no runtime dim argument. + kel::lbm::data<sch::MacroStruct<sch::T,sch::D2Q9::D,dim_x,dim_y>> macros; for(uint64_t step = 0; step < lattice_steps; ++step){ @@ -319,9 +330,21 @@ int main(){ apply_for_cells([&](auto& cell, std::size_t i, std::size_t j){ auto& dfs = cell.template get<"dfs">(); - typename kel::lbm::native_data_type<sch::T>::type rho; - std::array<typename kel::lbm::native_data_type<sch::T>::type, sch::D2Q9::D> vel; - compute_rho_u<sch::T,sch::D2Q9>(dfs,rho,vel); + // compute_rho_u takes a FixedArray of populations now, while this + // example still stores them in a Cell. Same Q values, different + // schema, so copy across rather than reinterpret. + kel::lbm::data<sch::FixedArray<sch::T,sch::D2Q9::Q>> dfs_arr; + for(uint64_t q{0u}; q < sch::D2Q9::Q; ++q){ + dfs_arr.at({q}) = dfs({q}); + } + + kel::lbm::data<sch::Scalar<sch::T>> rho_d; + kel::lbm::data<sch::Vector<sch::T,sch::D2Q9::D>> vel_d; + compute_rho_u<sch::T,sch::D2Q9>(dfs_arr,rho_d,vel_d); + + const auto rho = rho_d.at({}).get(); + const typename kel::lbm::native_data_type<sch::T>::type vel[sch::D2Q9::D] = { + vel_d.at({{0u}}).get(), vel_d.at({{1u}}).get()}; if(i > 1 && (i+2) < dim_x && j > 1 && (j+2) < dim_y){ sum += rho; @@ -404,14 +427,17 @@ int main(){ apply_for_cells([&](auto& cell, std::size_t i, std::size_t j){ auto dfs = even_step ? cell.template get<"dfs_old">() : cell.template get<"dfs">(); - auto& rho = macros.at({{i,j}}).template get<"pressure">(); - auto& vel = macros.at({{i,j}}).template get<"momentum">(); - compute_rho_u<sch::T,sch::D2Q9>(dfs,rho,vel); + kel::lbm::data<sch::FixedArray<sch::T,sch::D2Q9::Q>> dfs_arr; + for(uint64_t q{0u}; q < sch::D2Q9::Q; ++q){ + dfs_arr.at({q}) = dfs({q}); + } + + auto& rho = macros.template get<"density">().at({{i,j}}); + auto& vel = macros.template get<"velocity">().at({{i,j}}); + compute_rho_u<sch::T,sch::D2Q9>(dfs_arr,rho,vel); }, lattice); - std::string vtk_f_name{"tmp/cavity_2d_"}; - vtk_f_name += std::to_string(file_no) + ".vtk"; - write_vtk_file(vtk_f_name, macros); + write_vtk_file("tmp", "cavity_2d", file_no, macros); ++file_no; } diff --git a/examples/cavity_2d_gpu/cavity_2d_gpu.cpp b/examples/cavity_2d_gpu/cavity_2d_gpu.cpp index ab1ab40..55225d1 100644 --- a/examples/cavity_2d_gpu/cavity_2d_gpu.cpp +++ b/examples/cavity_2d_gpu/cavity_2d_gpu.cpp @@ -46,7 +46,7 @@ using CellStruct = CellFieldStruct< template<typename T, uint64_t D> using MacroStruct = Struct< - Member<FixedArray<T,D>, "momentum">, + Member<FixedArray<T,D>, "velocity">, Member<T, "pressure"> >; @@ -309,7 +309,9 @@ int main(){ {0.1} }; - print_lbm_meta<sch::T, sch::D2Q9>(conv, {1e-3}); + // Lid driven cavity: the lid velocity and the cavity height are the + // characteristic scales. + print_lbm_meta<sch::T, sch::D2Q9>(conv, {1e-3}, {0.1}, {dim_y}); auto eo_lbm_dir = output_directory(); if(eo_lbm_dir.is_error()){ @@ -361,7 +363,7 @@ int main(){ auto dfs_field = even_step ? dfs_old : dfs; auto& rho = macros.at(index).template get<"pressure">(); - auto& vel = macros.at(index).template get<"momentum">(); + auto& vel = macros.at(index).template get<"velocity">(); compute_rho_u<sch::T,sch::D2Q9>(dfs_field[j],rho,vel); }, {{0u,0u}}, meta); std::string vtk_f_name{"tmp/cavity_2d_gpu_"}; diff --git a/examples/cavity_3d.cpp b/examples/cavity_3d.cpp index e62c017..f38ca00 100644 --- a/examples/cavity_3d.cpp +++ b/examples/cavity_3d.cpp @@ -45,7 +45,7 @@ using CellStruct = Struct< template<typename T, uint64_t D> using MacroStruct = Struct< - Member<FixedArray<T,D>, "momentum">, + Member<FixedArray<T,D>, "velocity">, Member<T, "pressure"> >; @@ -410,7 +410,7 @@ int main(){ auto dfs = even_step ? cell.template get<"dfs_old">() : cell.template get<"dfs">(); auto& rho = macros.at({{i,j}}).template get<"pressure">(); - auto& vel = macros.at({{i,j}}).template get<"momentum">(); + auto& vel = macros.at({{i,j}}).template get<"velocity">(); compute_rho_u<sch::T,sch::D2Q9>(dfs,rho,vel); }, lattice); diff --git a/examples/dkt/common.hpp b/examples/dkt/common.hpp index 6c1dc44..abef548 100644 --- a/examples/dkt/common.hpp +++ b/examples/dkt/common.hpp @@ -56,7 +56,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity">, Member<VelChunk<T,Desc>, "force"> diff --git a/examples/dkt/init.hpp b/examples/dkt/init.hpp index c52078c..4a55dbc 100644 --- a/examples/dkt/init.hpp +++ b/examples/dkt/init.hpp @@ -57,7 +57,7 @@ kel::lbm::error_or<void> init( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -94,8 +94,8 @@ kel::lbm::error_or<void> init( ); { - kel::lbm::data<sch::Scalar<T>> radius_p; - radius_p.at({}).set(conv.meter_si_to_lbm({{0.05}}).handle().get()); + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; + radius_p.at({}).set(conv.meter_si_to_lbm({0.05}).handle().get()); std::cout<<"RADIUS: "<<radius_p.at({}).get()<<std::endl; kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); @@ -107,8 +107,8 @@ kel::lbm::error_or<void> init( auto& prb = p.template get<"rigid_body">(); auto& p_pos = prb.template get<"position">(); auto& p_posold = prb.template get<"position_old">(); - p_pos.at({{0u}}) = conv.meter_si_to_lbm({{0.16}}).handle(); - p_pos.at({{1u}}) = conv.meter_si_to_lbm({{0.2}}).handle(); + p_pos.at({{0u}}) = conv.meter_si_to_lbm({0.16}).handle(); + p_pos.at({{1u}}) = conv.meter_si_to_lbm({0.2}).handle(); p_posold = p_pos; } diff --git a/examples/dkt/sim.cpp b/examples/dkt/sim.cpp index 44fd145..c230aae 100644 --- a/examples/dkt/sim.cpp +++ b/examples/dkt/sim.cpp @@ -45,9 +45,9 @@ kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ converter<T> conv { // delta_x - {{0.41 / dim_y}}, + {0.41 / dim_y}, // delta_t - {{1.067e-4}} + {1.067e-4} }; print_lbm_meta<T,Desc>(conv,{1e-3},{1e-4},{0.5 * dim_y}); diff --git a/examples/dkt/step.hpp b/examples/dkt/step.hpp index 879def8..fa15609 100644 --- a/examples/dkt/step.hpp +++ b/examples/dkt/step.hpp @@ -76,7 +76,7 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Vector<T,Desc::D>> vel; { auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -152,7 +152,7 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Vector<T,Desc::D>> vel; { auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -228,7 +228,7 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Vector<T,Desc::D>> vel; { auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; diff --git a/examples/feature_development/common.hpp b/examples/feature_development/common.hpp index 6806b2c..e5274c2 100644 --- a/examples/feature_development/common.hpp +++ b/examples/feature_development/common.hpp @@ -48,7 +48,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity"> >; diff --git a/examples/feature_development/init.hpp b/examples/feature_development/init.hpp index a61ef37..333e9a3 100644 --- a/examples/feature_development/init.hpp +++ b/examples/feature_development/init.hpp @@ -57,7 +57,7 @@ kel::lbm::error_or<void> init( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -113,7 +113,7 @@ kel::lbm::error_or<void> init( ); { - kel::lbm::data<sch::Scalar<T>> radius_p; + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; radius_p.at({}).set(2); kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); diff --git a/examples/feature_development/sim.cpp b/examples/feature_development/sim.cpp index e852313..34ad5e3 100644 --- a/examples/feature_development/sim.cpp +++ b/examples/feature_development/sim.cpp @@ -46,9 +46,9 @@ kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/heterogeneous_computing/.nix/derivation.nix b/examples/heterogeneous_computing/.nix/derivation.nix index 48dfc7a..bf824bf 100644 --- a/examples/heterogeneous_computing/.nix/derivation.nix +++ b/examples/heterogeneous_computing/.nix/derivation.nix @@ -5,7 +5,8 @@ , forstio , pname , version -, kel-lbm +, adaptive-cpp +, kel }: stdenv.mkDerivation { @@ -30,7 +31,8 @@ stdenv.mkDerivation { forstio.remote-filesystem forstio.remote-io forstio.remote-sycl - kel-lbm.core + kel.lbm.core + adaptive-cpp ]; preferLocalBuild = true; diff --git a/examples/meta/meta.cpp b/examples/meta/meta.cpp index 6e0649f..3e573e5 100644 --- a/examples/meta/meta.cpp +++ b/examples/meta/meta.cpp @@ -28,10 +28,16 @@ template<typename T, typename Desc> kel::lbm::error_or<void> meta_lbm_info(const kel::lbm::data<kel::lbm::sch::MetaArgsStruct>& args_str){ using namespace kel::lbm; converter<T> conv{ - {args_str.template get<"dx">()}, - {args_str.template get<"dt">()} + as_unit<typename unit_division<sch::SiMeter<T>, sch::LbmMeter<T>>::Schema>( + args_str.template get<"dx">()), + as_unit<typename unit_division<sch::SiSecond<T>, sch::LbmSecond<T>>::Schema>( + args_str.template get<"dt">()) }; - print_lbm_meta<T,Desc>(conv, {args_str.template get<"kin_vis">()},{args_str.template get<"char_vel">()},{args_str.template get<"char_len">()}); + print_lbm_meta<T,Desc>( + conv, + as_unit<sch::SiKinematicViscosity<T>>(args_str.template get<"kin_vis">()), + as_unit<sch::SiVelocity<T>>(args_str.template get<"char_vel">()), + as_unit<sch::SiMeter<T>>(args_str.template get<"char_len">())); return kel::lbm::make_void(); } diff --git a/examples/moving_poiseulle_particles_2d_fplbm_gpu/sim.cpp b/examples/moving_poiseulle_particles_2d_fplbm_gpu/sim.cpp index d8c65fb..6d0dfb3 100644 --- a/examples/moving_poiseulle_particles_2d_fplbm_gpu/sim.cpp +++ b/examples/moving_poiseulle_particles_2d_fplbm_gpu/sim.cpp @@ -6,7 +6,6 @@ #include <forstio/remote/filesystem/easy.hpp> #include <forstio/codec/json/json.hpp> #include <forstio/codec/simple.hpp> -#include <forstio/codec/args.hpp> namespace kel { namespace lbm { @@ -54,7 +53,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity">, Member<VelChunk<T,Desc>, "force"> @@ -118,7 +117,7 @@ kel::lbm::error_or<void> init( auto& dfs_old_f = fields.template get<"dfs_old">(); auto& dfs_f = fields.template get<"dfs">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); auto& force_f = macros.template get<"force">(); @@ -145,7 +144,7 @@ kel::lbm::error_or<void> init( dfs_f.get_dims() ); - kel::lbm::data<sch::Scalar<T>> rad; + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad; rad.at({}) = dim_y * 0.1f; kel::lbm::data<sch::Scalar<T>> dense; dense.at({}) = 1.0f; @@ -159,7 +158,12 @@ kel::lbm::error_or<void> init( kel::lbm::data<sch::Vector<T,Desc::D>> zero; zero.at({{0u}}) = 0.0f; zero.at({{1u}}) = 0.0f; - rb.template get<"acceleration">() = zero; + // Acceleration is unit typed, so its zero has to be too. + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> zero_acc; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero_acc.at({{i}}).set(0); + } + rb.template get<"acceleration">() = zero_acc; rb.template get<"rotation">() = {}; rb.template get<"rotation_old">() = {}; rb.template get<"angular_acceleration">() = {}; @@ -174,11 +178,11 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, kel::lbm::data<sch::Ptr<sch::ParticleSpheroidGroup<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& particles, kel::lbm::data<sch::UInt64> t_i, + kel::lbm::data<sch::UInt64> sub_steps, device& dev ){ auto& q = dev.get_handle(); auto& info_f = fields.template get<"info">(); - auto& porous_f = macros.template get<"porosity">(); q.submit([&](acpp::sycl::handler& h){ component<T,Desc,cmpt::FpLbmReset,encode::Sycl<kel::lbm::enc::Native>> fplbm_reset; @@ -201,7 +205,7 @@ kel::lbm::error_or<void> step( index.at({{i}}).set(idx[i]); } - fplbm_one_part.apply(fields,macros,particles,index,t_i,{64u}); + fplbm_one_part.apply(fields,macros,particles,index,t_i,sub_steps); }); }).wait(); @@ -262,20 +266,21 @@ template<typename T, typename Desc> kel::lbm::error_or<void> lbm_main(int argc, char** argv){ using namespace kel::lbm; - using dfi = df_info<T,Desc>; - kel::lbm::data<sch::LbmArgs> args; { auto& args_n = args.template get<"args">(); auto& name = args_n.template get<"name">(); name.set("moving_poiseulle_particles_2d_fplbm_gpu"); + auto& sub_steps_arg = args_n.template get<"sub_steps">(); + sub_steps_arg.set(64u); } { - auto eov = saw::parse_args(args,argc,argv); + auto eov = kel::lbm::parse_args(args,argc,argv); if(eov.is_error()){ return eov; } } + kel::lbm::data<sch::UInt64> sub_steps = args.template get<"args">().template get<"sub_steps">(); auto eo_lbm_env = init_lbm_env(args); if(eo_lbm_env.is_error()){ return std::move(eo_lbm_env.get_error()); @@ -294,9 +299,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); @@ -378,7 +383,7 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ // BC + Collision { - auto eov = step<T,Desc>(lsd_view,lsdm_view,lsdp_view,i,dev); + auto eov = step<T,Desc>(lsd_view,lsdm_view,lsdp_view,i,sub_steps,dev); if(eov.is_error()){ return eov; } diff --git a/examples/moving_poiseulle_particles_2d_hlbm_gpu/sim.cpp b/examples/moving_poiseulle_particles_2d_hlbm_gpu/sim.cpp index 86f0ded..d96fa1b 100644 --- a/examples/moving_poiseulle_particles_2d_hlbm_gpu/sim.cpp +++ b/examples/moving_poiseulle_particles_2d_hlbm_gpu/sim.cpp @@ -17,6 +17,14 @@ constexpr uint64_t particle_amount = 1ul; namespace sch { +using LbmArgs = Args< + Struct< + Member<String, "name">, + Member<UInt64, "sub_steps"> + >, + Tuple<> +>; + using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y>; template<typename T, typename Desc> @@ -45,8 +53,9 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, + Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> >; @@ -66,7 +75,6 @@ kel::lbm::error_or<void> setup_initial_conditions( kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>>& particles ){ auto& info_f = fields.template get<"info">(); - auto& porous_f = macros.template get<"porosity">(); // Set everything as walls iterator<Desc::D>::apply( [&](auto& index){ @@ -108,7 +116,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -144,7 +152,7 @@ kel::lbm::error_or<void> setup_initial_conditions( {{1u,1u}} ); - kel::lbm::data<sch::Scalar<T>> rad; + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad; rad.at({}) = dim_y * 0.1f; kel::lbm::data<sch::Scalar<T>> dense; dense.at({}) = 1.0f; @@ -158,7 +166,12 @@ kel::lbm::error_or<void> setup_initial_conditions( kel::lbm::data<sch::Vector<T,Desc::D>> zero; zero.at({{0u}}) = 0.0f; zero.at({{1u}}) = 0.0f; - rb.template get<"acceleration">() = zero; + // Acceleration is unit typed, so its zero has to be too. + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> zero_acc; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero_acc.at({{i}}).set(0); + } + rb.template get<"acceleration">() = zero_acc; rb.template get<"rotation">() = {}; rb.template get<"rotation_old">() = {}; rb.template get<"angular_acceleration">() = {}; @@ -173,11 +186,11 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, kel::lbm::data<sch::Ptr<sch::ParticleSpheroidGroup<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& particles, kel::lbm::data<sch::UInt64> t_i, + kel::lbm::data<sch::UInt64> sub_steps, device& dev ){ auto& q = dev.get_handle(); auto& info_f = fields.template get<"info">(); - auto& porous_f = macros.template get<"porosity">(); q.submit([&](acpp::sycl::handler& h){ component<T,Desc,cmpt::HlbmReset,encode::Sycl<kel::lbm::enc::Native>> hlbm_reset; @@ -200,7 +213,7 @@ kel::lbm::error_or<void> step( index.at({{i}}).set(idx[i]); } - hlbm_one_part.apply(fields,macros,particles,index,t_i,{16u}); + hlbm_one_part.apply(fields,macros,particles,index,t_i,sub_steps); }); }).wait(); @@ -269,15 +282,28 @@ template<typename T, typename Desc> kel::lbm::error_or<void> lbm_main(int argc, char** argv){ using namespace kel::lbm; - using dfi = df_info<T,Desc>; - - auto eo_lbm_dir = output_directory(); - if(eo_lbm_dir.is_error()){ - return std::move(eo_lbm_dir.get_error()); + kel::lbm::data<sch::LbmArgs> args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("moving_poiseulle_particles_2d_hlbm_gpu"); + auto& sub_steps_arg = args_n.template get<"sub_steps">(); + sub_steps_arg.set(16u); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + kel::lbm::data<sch::UInt64> sub_steps = args.template get<"args">().template get<"sub_steps">(); + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); } - auto& lbm_dir = eo_lbm_dir.get_value(); + const auto& lbm_env = eo_lbm_env.get_value(); - auto out_dir = lbm_dir / "moving_poiseulle_particles_2d_hlbm_gpu"; + auto& out_dir = lbm_env.name_dir; { std::error_code ec; @@ -289,9 +315,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); @@ -373,7 +399,7 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ // BC + Collision { - auto eov = step<T,Desc>(lsd_view,lsdm_view,lsdp_view,i,dev); + auto eov = step<T,Desc>(lsd_view,lsdm_view,lsdp_view,i,sub_steps,dev); if(eov.is_error()){ return eov; } diff --git a/examples/moving_poiseulle_particles_2d_psm_gpu/sim.cpp b/examples/moving_poiseulle_particles_2d_psm_gpu/sim.cpp index a27c402..0bfefba 100644 --- a/examples/moving_poiseulle_particles_2d_psm_gpu/sim.cpp +++ b/examples/moving_poiseulle_particles_2d_psm_gpu/sim.cpp @@ -17,6 +17,14 @@ constexpr uint64_t particle_amount = 1ul; namespace sch { +using LbmArgs = Args< + Struct< + Member<String, "name">, + Member<UInt64, "sub_steps"> + >, + Tuple<> +>; + using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y>; template<typename T, typename Desc> @@ -45,7 +53,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity">, Member<VelChunk<T,Desc>, "force"> @@ -109,7 +117,7 @@ kel::lbm::error_or<void> init( auto& dfs_old_f = fields.template get<"dfs_old">(); auto& dfs_f = fields.template get<"dfs">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); auto& force_f = macros.template get<"force">(); @@ -136,7 +144,7 @@ kel::lbm::error_or<void> init( dfs_f.get_dims() ); - kel::lbm::data<sch::Scalar<T>> rad; + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad; rad.at({}) = dim_y * 0.1f; kel::lbm::data<sch::Scalar<T>> dense; dense.at({}) = 1.0f; @@ -150,7 +158,12 @@ kel::lbm::error_or<void> init( kel::lbm::data<sch::Vector<T,Desc::D>> zero; zero.at({{0u}}) = 0.0f; zero.at({{1u}}) = 0.0f; - rb.template get<"acceleration">() = zero; + // Acceleration is unit typed, so its zero has to be too. + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> zero_acc; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero_acc.at({{i}}).set(0); + } + rb.template get<"acceleration">() = zero_acc; rb.template get<"rotation">() = {}; rb.template get<"rotation_old">() = {}; rb.template get<"angular_acceleration">() = {}; @@ -165,11 +178,11 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, kel::lbm::data<sch::Ptr<sch::ParticleSpheroidGroup<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& particles, kel::lbm::data<sch::UInt64> t_i, + kel::lbm::data<sch::UInt64> sub_steps, device& dev ){ auto& q = dev.get_handle(); auto& info_f = fields.template get<"info">(); - auto& porous_f = macros.template get<"porosity">(); q.submit([&](acpp::sycl::handler& h){ component<T,Desc,cmpt::PsmReset,encode::Sycl<kel::lbm::enc::Native>> psm_reset; @@ -204,9 +217,9 @@ kel::lbm::error_or<void> step( } p_reset.apply(particles,index,t_i); - p_gather.apply(fields,macros,particles,index,t_i,{16u}); - p_contact.apply(particles,index,t_i,{16u}); - p_integrate.apply(particles,index,t_i,{16u}); + p_gather.apply(fields,macros,particles,index,t_i,sub_steps); + p_contact.apply(particles,index,t_i,sub_steps); + p_integrate.apply(particles,index,t_i,sub_steps); }); }).wait(); @@ -267,15 +280,28 @@ template<typename T, typename Desc> kel::lbm::error_or<void> lbm_main(int argc, char** argv){ using namespace kel::lbm; - using dfi = df_info<T,Desc>; - - auto eo_lbm_dir = output_directory(); - if(eo_lbm_dir.is_error()){ - return std::move(eo_lbm_dir.get_error()); + kel::lbm::data<sch::LbmArgs> args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("moving_poiseulle_particles_2d_psm_gpu"); + auto& sub_steps_arg = args_n.template get<"sub_steps">(); + sub_steps_arg.set(16u); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + kel::lbm::data<sch::UInt64> sub_steps = args.template get<"args">().template get<"sub_steps">(); + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); } - auto& lbm_dir = eo_lbm_dir.get_value(); + const auto& lbm_env = eo_lbm_env.get_value(); - auto out_dir = lbm_dir / "moving_poiseulle_particles_2d_psm_gpu"; + auto& out_dir = lbm_env.name_dir; { std::error_code ec; @@ -287,9 +313,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); @@ -371,7 +397,7 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ // BC + Collision { - auto eov = step<T,Desc>(lsd_view,lsdm_view,lsdp_view,i,dev); + auto eov = step<T,Desc>(lsd_view,lsdm_view,lsdp_view,i,sub_steps,dev); if(eov.is_error()){ return eov; } diff --git a/examples/one_particle_sedimentation_2d/common.hpp b/examples/one_particle_sedimentation_2d/common.hpp index e28622f..f5382e0 100644 --- a/examples/one_particle_sedimentation_2d/common.hpp +++ b/examples/one_particle_sedimentation_2d/common.hpp @@ -58,7 +58,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity">, Member<VelChunk<T,Desc>, "force">, diff --git a/examples/one_particle_sedimentation_2d/init.hpp b/examples/one_particle_sedimentation_2d/init.hpp index 989ec4f..94ad3db 100644 --- a/examples/one_particle_sedimentation_2d/init.hpp +++ b/examples/one_particle_sedimentation_2d/init.hpp @@ -37,7 +37,7 @@ kel::lbm::error_or<void> init( auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -74,8 +74,8 @@ kel::lbm::error_or<void> init( ); { - kel::lbm::data<sch::Scalar<T>> radius_p; - radius_p.at({}).set(conv.meter_si_to_lbm({{0.00125}}).handle().get()); + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; + radius_p.at({}).set(conv.meter_si_to_lbm({0.00125}).handle().get()); std::cout<<"RADIUS: "<<radius_p.at({}).get()<<std::endl; kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); @@ -87,8 +87,8 @@ kel::lbm::error_or<void> init( auto& prb = p.template get<"rigid_body">(); auto& p_pos = prb.template get<"position">(); auto& p_posold = prb.template get<"position_old">(); - p_pos.at({{0u}}) = conv.meter_si_to_lbm({{0.01}}).handle(); - p_pos.at({{1u}}) = conv.meter_si_to_lbm({{0.04}}).handle(); + p_pos.at({{0u}}) = conv.meter_si_to_lbm({0.01}).handle(); + p_pos.at({{1u}}) = conv.meter_si_to_lbm({0.04}).handle(); p_posold = p_pos; } diff --git a/examples/one_particle_sedimentation_2d/sim.cpp b/examples/one_particle_sedimentation_2d/sim.cpp index 7c8456f..c9fae4d 100644 --- a/examples/one_particle_sedimentation_2d/sim.cpp +++ b/examples/one_particle_sedimentation_2d/sim.cpp @@ -2,12 +2,13 @@ #include "init.hpp" #include "step.hpp" -#include <forstio/codec/args.hpp> /** * For deciding what parameters to use maybe? */ namespace args { +using namespace kel::lbm::sch; + using LbmArgsStruct = Struct< Member<String, "coupling"> @@ -45,16 +46,16 @@ kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ converter<T> conv { // delta_x - {{0.02 / dim_x}}, + {0.02 / dim_x}, // delta_t - {{3e-7}} + {3e-7} }; kel::lbm::data<sch::SiKinematicViscosity<T>> kin_vis_si{1e-3}; auto kin_vis_lbm = conv.kinematic_viscosity_si_to_lbm(kin_vis_si); auto tau = conv.template kinematic_viscosity_lbm_to_tau<Desc>(kin_vis_lbm); print_lbm_meta<T,Desc>(conv,kin_vis_si,{1e-4},{2}); - std::cout<<"Position: "<<conv.meter_si_to_lbm({{0.01}}).handle().get()<<", "<<conv.meter_si_to_lbm({{0.04}}).handle().get()<<std::endl; + std::cout<<"Position: "<<conv.meter_si_to_lbm({0.01}).handle().get()<<", "<<conv.meter_si_to_lbm({0.04}).handle().get()<<std::endl; // kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> meta{{dim_x,dim_y}}; auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); @@ -220,7 +221,7 @@ kel::lbm::error_or<void> k_main(int argc, char** argv){ kel::lbm::data<args::LbmArgs> args; { - auto eov = saw::parse_args(args,argc,argv); + auto eov = kel::lbm::parse_args(args,argc,argv); if(eov.is_error()){ return eov; } diff --git a/examples/one_particle_sedimentation_2d/step.hpp b/examples/one_particle_sedimentation_2d/step.hpp index de42dec..f28586a 100644 --- a/examples/one_particle_sedimentation_2d/step.hpp +++ b/examples/one_particle_sedimentation_2d/step.hpp @@ -90,7 +90,8 @@ kel::lbm::error_or<void> step( }).wait(); q.submit([&](acpp::sycl::handler& h){ - kel::lbm::data<sch::Vector<T,Desc::D>> gf; + // Gravity is an acceleration, which is what the integrator expects. + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> gf; gf.at({{1u}}) = conv.acceleration_si_to_lbm({-9.81}).handle(); component<T,Desc,cmpt::FpLbmOneParticle,encode::Sycl<kel::lbm::enc::Native>> fplbm_one_part{gf}; @@ -134,8 +135,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -210,8 +211,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; diff --git a/examples/particles_gpu/particles_gpu.cpp b/examples/particles_gpu/particles_gpu.cpp index f3edbf2..94ec5f1 100644 --- a/examples/particles_gpu/particles_gpu.cpp +++ b/examples/particles_gpu/particles_gpu.cpp @@ -30,7 +30,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ auto& old_pos = body.template get<"position_old">(); auto& acceleration = body.template get<"acceleration">(); auto& p_rad = part.template get<"collision">().template get<"radius">(); - p_rad = {0.4f}; + // Scalar<T> base, so the value goes through the element rather than a + // braced whole-unit assignment. + p_rad.at({}).set(0.4f); if(j.get() % 2u == 0) acceleration.at({{1u}}) = {-9.81}; @@ -41,7 +43,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ } } - kel::lbm::data<sch::Scalar<sch::Float32>> time_step; + // A timestep is a lattice time, which makes (pos - pos_old)/time_step a + // velocity rather than an unlabelled vector. + kel::lbm::data<sch::LbmSecond<sch::Scalar<sch::Float32>>> time_step; time_step.at({}).set(0.05f); for(kel::lbm::data<sch::UInt64> dt{0u}; dt < kel::lbm::data<sch::UInt64>{32ul}; ++dt){ diff --git a/examples/poiseulle_2d/poiseulle_2d.cpp b/examples/poiseulle_2d/poiseulle_2d.cpp index 7ade1ca..39fc65f 100644 --- a/examples/poiseulle_2d/poiseulle_2d.cpp +++ b/examples/poiseulle_2d/poiseulle_2d.cpp @@ -40,7 +40,7 @@ using CellStruct = Struct< template<typename T, uint64_t D> using MacroStruct = Struct< - Member<FixedArray<T,D>, "momentum">, + Member<FixedArray<T,D>, "velocity">, Member<T, "pressure"> >; @@ -374,7 +374,7 @@ int main(int argc, char** argv){ auto& dfs = even_step ? cell.template get<"dfs">() : cell.template get<"dfs_old">(); auto& rho = macros.at(index).template get<"pressure">(); - auto& vel = macros.at(index).template get<"momentum">(); + auto& vel = macros.at(index).template get<"velocity">(); compute_rho_u<sch::T,sch::D2Q9>(dfs,rho,vel); rho = rho * kel::lbm::data<sch::T>{dfi::cs2}; diff --git a/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp b/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp index bece18f..5959506 100644 --- a/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp +++ b/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp @@ -35,7 +35,7 @@ using CellStruct = Struct< >; using MacroStruct = Struct< - Member<FixedArray<Float64,D2Q9::D>, "momentum">, + Member<FixedArray<Float64,D2Q9::D>, "velocity">, Member<Float64, "pressure"> >; @@ -244,7 +244,7 @@ void step( auto& macro_c = macro_cells[acc_id]; kel::lbm::data<sch::T>& rho = macro_c.template get<"pressure">(); - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"momentum">(); + kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"velocity">(); compute_rho_u<sch::T,Desc>(dfs_old,rho,vel); auto eq = equilibrium<sch::T,Desc>(rho,vel); @@ -261,7 +261,7 @@ void step( auto& macro_c = macro_cells[acc_id]; kel::lbm::data<sch::T>& rho = macro_c.template get<"pressure">(); - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"momentum">(); + kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"velocity">(); compute_rho_u<sch::T,Desc>(dfs_old,rho,vel); auto eq = equilibrium<sch::T,Desc>(rho,vel); @@ -278,7 +278,7 @@ void step( auto& macro_c = macro_cells[acc_id]; kel::lbm::data<sch::T>& rho = macro_c.template get<"pressure">(); - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"momentum">(); + kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"velocity">(); compute_rho_u<sch::T,Desc>(dfs_old,rho,vel); auto eq = equilibrium<sch::T,Desc>(rho,vel); @@ -341,9 +341,9 @@ kel::lbm::error_or<void> kel_main(int argc, char** argv){ // lbm::converter<lbm::sch::Float64> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; uint64_t x_d = 1024u; diff --git a/examples/poiseulle_3d/.nix/derivation.nix b/examples/poiseulle_3d/.nix/derivation.nix index d53b5fe..b090add 100644 --- a/examples/poiseulle_3d/.nix/derivation.nix +++ b/examples/poiseulle_3d/.nix/derivation.nix @@ -5,7 +5,7 @@ , forstio , pname , version -, kel-lbm +, kel , adaptive-cpp }: @@ -28,7 +28,7 @@ stdenv.mkDerivation { forstio.remote forstio.remote-sycl adaptive-cpp - kel-lbm.core + kel.lbm.core ]; preferLocalBuild = true; diff --git a/examples/poiseulle_3d_gpu/sim.cpp b/examples/poiseulle_3d_gpu/sim.cpp index 3fe7358..83ba309 100644 --- a/examples/poiseulle_3d_gpu/sim.cpp +++ b/examples/poiseulle_3d_gpu/sim.cpp @@ -45,7 +45,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<MomChunk<T,Desc>, "momentum">, + Member<MomChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity"> >; @@ -100,7 +100,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& mom_f = macros.template get<"momentum">(); + auto& mom_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -155,7 +155,7 @@ kel::lbm::error_or<void> step( // auto coll_ev = q.submit([&](acpp::sycl::handler& h){ // Need nicer things to handle the flow. I see improvement here - component<T,Desc,cmpt::BGK, encode::Sycl<kel::lbm::enc::Native>> collision{conv.template kinematic_viscosity_si_to_tau<Desc>({0.1}).handle()}; + component<T,Desc,cmpt::BGK, encode::Sycl<kel::lbm::enc::Native>> collision{conv.template kinematic_viscosity_si_to_tau<Desc>({0.1})}; // component<T,Desc,cmpt::HLBM,encode::Sycl<kel::lbm::enc::Native>> collision{0.6}; component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; @@ -254,9 +254,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{0.01},{1024.0}); diff --git a/examples/poiseulle_3d_gpu_minimal/.nix/derivation.nix b/examples/poiseulle_3d_gpu_minimal/.nix/derivation.nix new file mode 100644 index 0000000..98ad46a --- /dev/null +++ b/examples/poiseulle_3d_gpu_minimal/.nix/derivation.nix @@ -0,0 +1,41 @@ +{ lib +, stdenv +, scons +, clang-tools +, forstio +, python3 +, pname +, version +, adaptive-cpp +, kel +}: + +stdenv.mkDerivation { + pname = pname + "-examples-" + "poiseulle_3d_gpu_minimal"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + python3 + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + forstio.codec-unit + forstio.io + forstio.remote + forstio.remote-filesystem + forstio.codec-json + adaptive-cpp + kel.lbm.core + kel.lbm.sycl + ]; + + preferLocalBuild = true; + + outputs = [ "out" "dev" ]; +} diff --git a/examples/poiseulle_3d_gpu_minimal/SConscript b/examples/poiseulle_3d_gpu_minimal/SConscript new file mode 100644 index 0000000..29f73db --- /dev/null +++ b/examples/poiseulle_3d_gpu_minimal/SConscript @@ -0,0 +1,34 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +examples_env = env.Clone(); +examples_env['CXX'] = 'syclcc-clang'; +examples_env['CXXFLAGS'] += ['-O3']; + +examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += examples_env.sources; +env.headers += examples_env.headers; + +# Cavity2D +examples_objects = []; +examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); +examples_env.poiseulle_2d_gpu = examples_env.Program('#bin/poiseulle_3d_gpu_minimal', [examples_objects]); + +# Set Alias +env.examples = [ + examples_env.poiseulle_2d_gpu +]; +env.Alias('examples', env.examples); +env.targets += ['examples']; +env.Install('$prefix/bin/', env.examples); diff --git a/examples/poiseulle_3d_gpu_minimal/SConstruct b/examples/poiseulle_3d_gpu_minimal/SConstruct new file mode 100644 index 0000000..0611b67 --- /dev/null +++ b/examples/poiseulle_3d_gpu_minimal/SConstruct @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env_vars.Add('build_examples', + help='If examples should be built', + default="true" +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=[ + '-std=c++20', + '-g', + '-Wall', + '-Wextra' + ], + LIBS=[ + 'forstio-core', + 'forstio-async', + 'forstio-io' + ] +); +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/examples/poiseulle_3d_gpu_minimal/sim.cpp b/examples/poiseulle_3d_gpu_minimal/sim.cpp new file mode 100644 index 0000000..0bc4642 --- /dev/null +++ b/examples/poiseulle_3d_gpu_minimal/sim.cpp @@ -0,0 +1,433 @@ +#include <kel/lbm/sycl/lbm.hpp> +#include <kel/lbm/lbm.hpp> +#include <kel/lbm/particle.hpp> + +#include <forstio/io/io.hpp> +#include <forstio/remote/filesystem/easy.hpp> +#include <forstio/codec/json/json.hpp> +#include <forstio/codec/simple.hpp> + +namespace kel { +namespace lbm { + +// Deliberately tiny: 32 x 16 x 16 = 8192 cells, so a full run finishes in +// seconds and the inlet signal reaches the outlet in ~32 steps. The full size +// case lives in poiseulle_3d_gpu. +constexpr uint64_t dim_x = 32ul; +constexpr uint64_t dim_y = 16ul; +constexpr uint64_t dim_z = 16ul; +constexpr uint64_t particle_size = 4ul; + +namespace sch { + +using LbmArgs = Args< + Struct< + Member<String, "name">, + Member<UInt64, "time_steps"> + >, + Tuple<> +>; + +using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using DfChunk = Chunk<FixedArray<T,Desc::Q>, 1u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using ScalarChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using VectorChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using ChunkStruct = Struct< + Member<InfoChunk, "info">, + Member<DfChunk<T,Desc>, "dfs">, + Member<DfChunk<T,Desc>, "dfs_old">, + Member<VectorChunk<T,Desc>, "particle_N">, + Member<ScalarChunk<T,Desc>, "particle_D"> +>; + +template<typename T, typename Desc> +using MomChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; + +template<typename T> +using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using MacroStruct = Struct< + Member<MomChunk<T,Desc>, "velocity">, + Member<RhoChunk<T>, "density">, + Member<ScalarChunk<T,Desc>, "porosity"> +>; +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> setup_initial_conditions( + converter<T>& conv, + kel::lbm::data<sch::ChunkStruct<T,Desc>>& fields, + kel::lbm::data<sch::MacroStruct<T,Desc>>& macros, + kel::lbm::data<sch::FixedArray<sch::Particle<T,Desc::D>, particle_size>>& particles +){ + // Unit conversion is not exercised in this reduced case. + (void) conv; + + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u,1u}} + ); + + // Inflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u,0u}}, + {{1u,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + + // Outflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u,0u}}, + {{dim_x,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& mom_f = macros.template get<"velocity">(); + auto& por_f = macros.template get<"porosity">(); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + por_f.at(index).at({}) = {1}; + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + auto eq = equilibrium<T,Desc>(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims() + ); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + mom.at({{0u}}) = 0.01; + auto eq = equilibrium<T,Desc>(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u,1u}} + ); + + // No particles in this case -- the array is carried only so the schema + // matches poiseulle_3d_gpu. + (void) particles; + + return kel::lbm::make_void(); +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> step( + converter<T>& conv, + kel::lbm::data<sch::Ptr<sch::ChunkStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& fields, + kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, + kel::lbm::data<sch::UInt64> t_i, + device& dev +){ + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); + + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + // Need nicer things to handle the flow. I see improvement here + component<T,Desc,cmpt::BGK, encode::Sycl<kel::lbm::enc::Native>> collision{conv.template kinematic_viscosity_si_to_tau<Desc>({0.1})}; + // component<T,Desc,cmpt::HLBM,encode::Sycl<kel::lbm::enc::Native>> collision{0.6}; + component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; + + component<T,Desc,cmpt::ZouHeHorizontal<true>,encode::Sycl<kel::lbm::enc::Native>> flow_in{ + [&](){ + uint64_t target_t_i = 256u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.001 / target_t_i) * t_i.get(); + } + return 1.001; + }() + }; + component<T,Desc,cmpt::ZouHeHorizontal<false>,encode::Sycl<kel::lbm::enc::Native>> flow_out{1.0}; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: + break; + case 1u: + bb.apply(fields,index,t_i); + break; + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 3u: + flow_in.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: + break; + } + }); + }).wait(); + + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::Stream,encode::Sycl<kel::lbm::enc::Native>> stream; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(fields,index,t_i); + } + }); + }).wait(); + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ + + return kel::lbm::make_void(); +} +} +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> lbm_main(int argc, char** argv){ + using namespace kel::lbm; + + kel::lbm::data<sch::LbmArgs> args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("poiseulle_3d_gpu_minimal"); + auto& steps_arg = args_n.template get<"time_steps">(); + steps_arg.set(64u); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); + } + const auto& lbm_env = eo_lbm_env.get_value(); + + auto& out_dir = lbm_env.name_dir; + + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not create output directory"); + } + } + + converter<T> conv { + // delta_x + {1.0}, + // delta_t + {1.0} + }; + + print_lbm_meta<T,Desc>(conv,{0.1},{0.01},{1024.0}); + + auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); + auto lbm_macro_data_ptr = kel::lbm::heap<kel::lbm::data<sch::MacroStruct<T,Desc>>>(); + auto lbm_particle_data_ptr = kel::lbm::heap<kel::lbm::data<sch::FixedArray<sch::Particle<T,Desc::D>, particle_size>>>(); + + std::cout<<"Estimated Bytes: "<<memory_estimate<sch::ChunkStruct<T,Desc>,sch::MacroStruct<T,Desc>>().get()<<std::endl; + + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error<kel::lbm::err::critical>( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; + + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); + + device dev; + + auto& sycl_q = dev.get_handle(); + + sycl_q.wait(); + { + auto eov = setup_initial_conditions<T,Desc>(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + kel::lbm::data<sch::ChunkStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_data{sycl_q}; + kel::lbm::data<sch::MacroStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_macro_data{sycl_q}; + kel::lbm::data<sch::FixedArray<sch::Particle<T,Desc::D>, particle_size>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_particle_data{sycl_q}; + sycl_q.wait(); + + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + kel::lbm::data<sch::UInt64> time_steps = args.template get<"args">().template get<"time_steps">(); + + for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ + { + { + auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + /*{ + std::string file_name = "p_"; + file_name += std::to_string(i.get()); + file_name += ".json"; + auto eov = saw::easy::encode_and_write_file<sch::FixedArray<sch::Particle<T,Desc::D>,particle_size>,saw::encode::Json>(out_dir, *lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + }*/ + } + { + auto eov = step<T,Desc>(conv,lsd_view,lsdm_view,i,dev); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + wait.poll(); + if(print_status){ + std::cout<<"Status: "<<i.get()<<" of "<<time_steps.get()<<" - "<<(i.template cast_to<sch::Float64>().get() * 100 / time_steps.get())<<"%"<<std::endl; + print_status = false; + } + } + sycl_q.wait(); + { + auto eov = write_vtk_file(out_dir,"t",time_steps.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + /* + iterator<Desc::D>::apply( + [&](auto& index){ + std::cout<<index.at({0u}).get()<<" "<<index.at({1u}).get()<<" "<<lbm_data.get<"info">().at(index).template cast_to<sch::UInt16>().get()<<"\n"; + }, + {{0u,0u}}, + {{dim_x, dim_y}} + ); + */ + + sycl_q.wait(); + return kel::lbm::make_void(); +} + +using FloatT = kel::lbm::sch::Float32; + +int main(int argc, char** argv){ + auto eov = lbm_main<FloatT,kel::lbm::sch::D3Q27>(argc, argv); + if(eov.is_error()){ + auto& err = eov.get_error(); + std::cerr<<"[Error] "<<err.get_category(); + auto err_msg = err.get_message(); + if(err_msg.size() > 0u){ + std::cerr<<" - "<<err_msg; + } + std::cerr<<std::endl; + return err.get_id(); + } + return 0; +} diff --git a/examples/poiseulle_moving_particle_2d_psm_gpu/sim.cpp b/examples/poiseulle_moving_particle_2d_psm_gpu/sim.cpp index 793be1f..7e8b353 100644 --- a/examples/poiseulle_moving_particle_2d_psm_gpu/sim.cpp +++ b/examples/poiseulle_moving_particle_2d_psm_gpu/sim.cpp @@ -46,7 +46,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity"> >; @@ -122,7 +122,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -261,9 +261,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/poiseulle_particle_3d_hlbm_gpu/.nix/derivation.nix b/examples/poiseulle_particle_3d_hlbm_gpu/.nix/derivation.nix new file mode 100644 index 0000000..3f1dd7b --- /dev/null +++ b/examples/poiseulle_particle_3d_hlbm_gpu/.nix/derivation.nix @@ -0,0 +1,41 @@ +{ lib +, stdenv +, scons +, clang-tools +, forstio +, python3 +, pname +, version +, adaptive-cpp +, kel +}: + +stdenv.mkDerivation { + pname = pname + "-examples-" + "poiseulle_particle_3d_hlbm_gpu"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + python3 + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + forstio.codec-unit + forstio.io + forstio.remote + forstio.remote-filesystem + forstio.codec-json + adaptive-cpp + kel.lbm.core + kel.lbm.sycl + ]; + + preferLocalBuild = true; + + outputs = [ "out" "dev" ]; +} diff --git a/examples/poiseulle_particle_3d_hlbm_gpu/SConscript b/examples/poiseulle_particle_3d_hlbm_gpu/SConscript new file mode 100644 index 0000000..478ee44 --- /dev/null +++ b/examples/poiseulle_particle_3d_hlbm_gpu/SConscript @@ -0,0 +1,34 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +examples_env = env.Clone(); +examples_env['CXX'] = 'syclcc-clang'; +examples_env['CXXFLAGS'] += ['-O3']; + +examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += examples_env.sources; +env.headers += examples_env.headers; + +# Cavity2D +examples_objects = []; +examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); +examples_env.poiseulle_2d_gpu = examples_env.Program('#bin/poiseulle_particle_3d_hlbm_gpu', [examples_objects]); + +# Set Alias +env.examples = [ + examples_env.poiseulle_2d_gpu +]; +env.Alias('examples', env.examples); +env.targets += ['examples']; +env.Install('$prefix/bin/', env.examples); diff --git a/examples/poiseulle_particle_3d_hlbm_gpu/SConstruct b/examples/poiseulle_particle_3d_hlbm_gpu/SConstruct new file mode 100644 index 0000000..0611b67 --- /dev/null +++ b/examples/poiseulle_particle_3d_hlbm_gpu/SConstruct @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env_vars.Add('build_examples', + help='If examples should be built', + default="true" +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=[ + '-std=c++20', + '-g', + '-Wall', + '-Wextra' + ], + LIBS=[ + 'forstio-core', + 'forstio-async', + 'forstio-io' + ] +); +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/examples/poiseulle_particle_3d_hlbm_gpu/sim.cpp b/examples/poiseulle_particle_3d_hlbm_gpu/sim.cpp new file mode 100644 index 0000000..59ccd46 --- /dev/null +++ b/examples/poiseulle_particle_3d_hlbm_gpu/sim.cpp @@ -0,0 +1,604 @@ +#include <kel/lbm/sycl/lbm.hpp> +#include <kel/lbm/lbm.hpp> +#include <kel/lbm/particle.hpp> + +#include <fstream> + +#include <forstio/io/io.hpp> +#include <forstio/remote/filesystem/easy.hpp> +#include <forstio/codec/json/json.hpp> +#include <forstio/codec/simple.hpp> + +namespace kel { +namespace lbm { + +// Deliberately tiny: 32 x 32 x 32 = 32768 cells, so a full run finishes in +// seconds. Square in y/z so the channel cross section is symmetric about the +// particle, which makes a stray transverse force obvious. +constexpr uint64_t dim_x = 32ul; +constexpr uint64_t dim_y = 32ul; +constexpr uint64_t dim_z = 32ul; + +constexpr uint64_t particle_amount = 1ul; + +/** + * The particle lives in lattice space, exactly like the fluid: the radius is a + * number of cells, the density is relative to the fluid's rho = 1, and the + * mass, inertia, force and torque that follow are all lattice quantities. The + * converter handles the fluid's SI <-> LBM bookkeeping and is deliberately not + * applied to any of this -- so every column of particle.csv is in cells, + * steps, and lattice mass. + */ +constexpr double particle_radius = 0.15 * static_cast<double>(dim_y); +constexpr double particle_density = 1.0; + +namespace sch { + +using LbmArgs = Args< + Struct< + Member<String, "name">, + Member<UInt64, "time_steps">, + Member<UInt64, "sub_steps"> + >, + Tuple<> +>; + +using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using DfChunk = Chunk<FixedArray<T,Desc::Q>, 1u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using ScalarChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using VectorChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using ChunkStruct = Struct< + Member<InfoChunk, "info">, + Member<DfChunk<T,Desc>, "dfs">, + Member<DfChunk<T,Desc>, "dfs_old">, + Member<VectorChunk<T,Desc>, "particle_N">, + Member<ScalarChunk<T,Desc>, "particle_D"> +>; + +template<typename T, typename Desc> +using MomChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; + +template<typename T> +using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; + +/** + * "force" is not optional here: HlbmOneParticleGather reads it, so the 3D + * macro struct needs it the moment a particle is carried. + */ +template<typename T, typename Desc> +using MacroStruct = Struct< + Member<MomChunk<T,Desc>, "velocity">, + Member<RhoChunk<T>, "density">, + Member<MomChunk<T,Desc>, "force">, + Member<ScalarChunk<T,Desc>, "porosity"> +>; + +/** + * particle_aabb is specialised only on ParticleGroup, so a bare + * FixedArray<Particle<...>> hits its always_false primary template. + */ +template<typename T, typename Desc> +using ParticleSpheroidGroup = ParticleGroup<T,Desc::D,particle_amount,coll::Spheroid<T>>; +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> setup_initial_conditions( + converter<T>& conv, + kel::lbm::data<sch::ChunkStruct<T,Desc>>& fields, + kel::lbm::data<sch::MacroStruct<T,Desc>>& macros, + kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>>& particles +){ + // Unit conversion is not exercised in this reduced case. + (void) conv; + + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u,1u}} + ); + + // Inflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u,0u}}, + {{1u,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + + // Outflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u,0u}}, + {{dim_x,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& mom_f = macros.template get<"velocity">(); + auto& por_f = macros.template get<"porosity">(); + auto& force_f = macros.template get<"force">(); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + por_f.at(index).at({}) = {1}; + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + auto eq = equilibrium<T,Desc>(rho,mom); + + df = eq; + + auto& force = force_f.at(index); + for(uint64_t i{0u}; i < Desc::D; ++i){ + force.at({{i}}).set(0); + } + }, + {},// 0-index + df_f.get_dims() + ); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + mom.at({{0u}}) = 0.01; + auto eq = equilibrium<T,Desc>(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u,1u}} + ); + + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad; + rad.at({}).set(particle_radius); + kel::lbm::data<sch::Scalar<T>> dense; + dense.at({}).set(particle_density); + + // Mask resolution 16 in 3D is 16^3 = 4096 voxels. The analytic spheroid + // porosity is what HLBM actually calls, but the mask is still filled. + particles = create_spheroid_particle_group<T,Desc::D,particle_amount>(rad,dense,{16u}); + { + auto& rb = particles.template get<"particles">().at({0u}).template get<"rigid_body">(); + auto& rbp = rb.template get<"position">(); + rbp.at({{0u}}) = 0.3 * dim_x; + rbp.at({{1u}}) = 0.5 * dim_y; + rbp.at({{2u}}) = 0.5 * dim_z; + + // position_old == position means the body starts at rest. It also + // makes a pinned body exact later: 2x - x_old + 0 == x. + rb.template get<"position_old">() = rbp; + + kel::lbm::data<sch::Vector<T,Desc::D>> zero; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero.at({{i}}).set(0); + } + // Acceleration is unit typed now, so its zero has to be too. + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> zero_acc; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero_acc.at({{i}}).set(0); + } + rb.template get<"acceleration">() = zero_acc; + + // In 3D these are Vector<T,3> (a rotation vector), not a scalar angle. + rb.template get<"rotation">() = zero; + rb.template get<"rotation_old">() = zero; + rb.template get<"angular_acceleration">() = zero; + } + + return kel::lbm::make_void(); +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> step( + converter<T>& conv, + kel::lbm::data<sch::Ptr<sch::ChunkStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& fields, + kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, + kel::lbm::data<sch::Ptr<sch::ParticleSpheroidGroup<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& particles, + kel::lbm::data<sch::UInt64> t_i, + kel::lbm::data<sch::UInt64> sub_steps, + device& dev +){ + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); + + // Porosity back to 1 (all fluid) before the particle restamps itself. + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::HlbmReset,encode::Sycl<kel::lbm::enc::Native>> hlbm_reset; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + hlbm_reset.apply(fields,macros,index,t_i); + }); + }).wait(); + + /** + * One work item per particle. The stamp has to come after the integrate: + * it writes the post-integration surface velocity back onto the lattice. + */ + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::ParticleForceReset,encode::Sycl<kel::lbm::enc::Native>> p_reset; + component<T,Desc,cmpt::HlbmOneParticleGather,encode::Sycl<kel::lbm::enc::Native>> p_gather; + component<T,Desc,cmpt::ParticleIntegrate,encode::Sycl<kel::lbm::enc::Native>> p_integrate; + component<T,Desc,cmpt::HlbmOneParticleStamp,encode::Sycl<kel::lbm::enc::Native>> p_stamp; + + // Soft shell against the domain walls so the sphere cannot tunnel out. + dem_params<T> p_dem{1.0e-2, 0.3, 0.0, 0.0}; + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> p_extent{{dim_x,dim_y,dim_z}}; + component<T,Desc,cmpt::ParticleContact,encode::Sycl<kel::lbm::enc::Native>> p_contact{p_dem,p_extent}; + + h.parallel_for(acpp::sycl::range<1u>{particle_amount}, [=](acpp::sycl::id<1u> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,1u>> index; + for(uint64_t i = 0u; i < 1u; ++i){ + index.at({{i}}).set(idx[i]); + } + + p_reset.apply(particles,index,t_i); + p_gather.apply(fields,macros,particles,index,t_i,sub_steps); + p_contact.apply(particles,index,t_i,sub_steps); + p_integrate.apply(particles,index,t_i,sub_steps); + p_stamp.apply(fields,macros,particles,index,t_i,sub_steps); + }); + }).wait(); + + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + /** + * Hlbm rather than BGK: it is the collision that consumes the + * porosity and particle_N/particle_D the stamp pass writes. + */ + component<T,Desc,cmpt::Hlbm, encode::Sycl<kel::lbm::enc::Native>> collision{conv.template kinematic_viscosity_si_to_tau<Desc>({0.1})}; + component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; + + component<T,Desc,cmpt::ZouHeHorizontal<true>,encode::Sycl<kel::lbm::enc::Native>> flow_in{ + [&](){ + uint64_t target_t_i = 256u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.001 / target_t_i) * t_i.get(); + } + return 1.001; + }() + }; + component<T,Desc,cmpt::ZouHeHorizontal<false>,encode::Sycl<kel::lbm::enc::Native>> flow_out{1.0}; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: + break; + case 1u: + bb.apply(fields,index,t_i); + break; + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 3u: + flow_in.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: + break; + } + }); + }).wait(); + + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::Stream,encode::Sycl<kel::lbm::enc::Native>> stream; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(fields,index,t_i); + } + }); + }).wait(); + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ + + return kel::lbm::make_void(); +} +} +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> lbm_main(int argc, char** argv){ + using namespace kel::lbm; + + kel::lbm::data<sch::LbmArgs> args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("poiseulle_particle_3d_hlbm_gpu"); + auto& steps_arg = args_n.template get<"time_steps">(); + steps_arg.set(256u); + auto& sub_arg = args_n.template get<"sub_steps">(); + sub_arg.set(16u); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); + } + const auto& lbm_env = eo_lbm_env.get_value(); + + auto& out_dir = lbm_env.name_dir; + + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not create output directory"); + } + } + + converter<T> conv { + // delta_x + {1.0}, + // delta_t + {1.0} + }; + + // Characteristic length is the body diameter -- the scale that sets Re here. + print_lbm_meta<T,Desc>(conv,{0.1},{0.01},{2.0 * particle_radius}); + + auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); + auto lbm_macro_data_ptr = kel::lbm::heap<kel::lbm::data<sch::MacroStruct<T,Desc>>>(); + auto lbm_particle_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>>>(); + + std::cout<<"Estimated Bytes: "<<memory_estimate<sch::ChunkStruct<T,Desc>,sch::MacroStruct<T,Desc>>().get()<<std::endl; + + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error<kel::lbm::err::critical>( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; + + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); + + device dev; + + auto& sycl_q = dev.get_handle(); + + sycl_q.wait(); + { + auto eov = setup_initial_conditions<T,Desc>(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + kel::lbm::data<sch::ChunkStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_data{sycl_q}; + kel::lbm::data<sch::MacroStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_macro_data{sycl_q}; + kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_particle_data{sycl_q}; + sycl_q.wait(); + + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + auto lsdp_view = make_view(lbm_sycl_particle_data); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + kel::lbm::data<sch::UInt64> time_steps = args.template get<"args">().template get<"time_steps">(); + kel::lbm::data<sch::UInt64> sub_steps = args.template get<"args">().template get<"sub_steps">(); + + /** + * Particle time series. write_vtk_file only accepts a struct of chunks, so + * a particle group cannot go through it -- a CSV beside the VTK output is + * the simpler answer, and it is what you actually plot. + * + * Velocity and angular velocity are not stored on the body: the scheme is + * position-Verlet, so both are recovered as (now - previous) * steps. + */ + std::ofstream particle_csv{out_dir / "particle.csv"}; + if(not particle_csv.is_open()){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not open particle.csv"); + } + particle_csv + <<"step,x,y,z,vx,vy,vz,fx,fy,fz," + <<"rot_x,rot_y,rot_z,omega_x,omega_y,omega_z," + <<"alpha_x,alpha_y,alpha_z,tau_x,tau_y,tau_z\n"; + + auto write_particle_row = [&](uint64_t step_i){ + auto& rb = lbm_particle_data_ptr->template get<"particles">() + .at({0u}).template get<"rigid_body">(); + + auto& pos = rb.template get<"position">(); + auto& pos_old = rb.template get<"position_old">(); + auto& force = rb.template get<"force">(); + auto& rot = rb.template get<"rotation">(); + auto& rot_old = rb.template get<"rotation_old">(); + auto& ang_acc = rb.template get<"angular_acceleration">(); + auto& torque = rb.template get<"torque">(); + + const auto steps = static_cast<double>(particle_sub_steps(sub_steps)); + + particle_csv<<step_i; + for(uint64_t k{0u}; k < Desc::D; ++k){ + particle_csv<<","<<pos.at({{k}}).get(); + } + for(uint64_t k{0u}; k < Desc::D; ++k){ + particle_csv<<","<<(pos.at({{k}}).get() - pos_old.at({{k}}).get()) * steps; + } + for(uint64_t k{0u}; k < Desc::D; ++k){ + particle_csv<<","<<force.at({{k}}).get(); + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<rot.at({{k}}).get(); + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<(rot.at({{k}}).get() - rot_old.at({{k}}).get()) * steps; + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<ang_acc.at({{k}}).get(); + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<torque.at({{k}}).get(); + } + particle_csv<<"\n"; + }; + + for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ + { + { + auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + { + auto eov = step<T,Desc>(conv,lsd_view,lsdm_view,lsdp_view,i,sub_steps,dev); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_particle_data,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + write_particle_row(i.get()); + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + wait.poll(); + if(print_status){ + std::cout<<"Status: "<<i.get()<<" of "<<time_steps.get()<<" - "<<(i.template cast_to<sch::Float64>().get() * 100 / time_steps.get())<<"%"<<std::endl; + print_status = false; + } + } + sycl_q.wait(); + { + auto eov = write_vtk_file(out_dir,"t",time_steps.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + /* + iterator<Desc::D>::apply( + [&](auto& index){ + std::cout<<index.at({0u}).get()<<" "<<index.at({1u}).get()<<" "<<lbm_data.get<"info">().at(index).template cast_to<sch::UInt16>().get()<<"\n"; + }, + {{0u,0u}}, + {{dim_x, dim_y}} + ); + */ + + sycl_q.wait(); + return kel::lbm::make_void(); +} + +using FloatT = kel::lbm::sch::Float32; + +int main(int argc, char** argv){ + auto eov = lbm_main<FloatT,kel::lbm::sch::D3Q27>(argc, argv); + if(eov.is_error()){ + auto& err = eov.get_error(); + std::cerr<<"[Error] "<<err.get_category(); + auto err_msg = err.get_message(); + if(err_msg.size() > 0u){ + std::cerr<<" - "<<err_msg; + } + std::cerr<<std::endl; + return err.get_id(); + } + return 0; +} diff --git a/examples/poiseulle_particles_2d_bgk_gpu/sim.cpp b/examples/poiseulle_particles_2d_bgk_gpu/sim.cpp index e8ae50a..dcc7c77 100644 --- a/examples/poiseulle_particles_2d_bgk_gpu/sim.cpp +++ b/examples/poiseulle_particles_2d_bgk_gpu/sim.cpp @@ -44,7 +44,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density"> >; @@ -101,7 +101,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); iterator<Desc::D>::apply( [&](auto& index){ @@ -270,9 +270,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/poiseulle_particles_2d_fplbm_gpu/sim.cpp b/examples/poiseulle_particles_2d_fplbm_gpu/sim.cpp index 9d9e32e..06c1d21 100644 --- a/examples/poiseulle_particles_2d_fplbm_gpu/sim.cpp +++ b/examples/poiseulle_particles_2d_fplbm_gpu/sim.cpp @@ -48,7 +48,7 @@ using ForceChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> @@ -108,7 +108,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -258,9 +258,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/poiseulle_particles_2d_gpu/common.hpp b/examples/poiseulle_particles_2d_gpu/common.hpp index 6806b2c..f4c9fed 100644 --- a/examples/poiseulle_particles_2d_gpu/common.hpp +++ b/examples/poiseulle_particles_2d_gpu/common.hpp @@ -48,8 +48,10 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, + // HlbmOneParticleGather reads this one. + Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> >; diff --git a/examples/poiseulle_particles_2d_gpu/init.hpp b/examples/poiseulle_particles_2d_gpu/init.hpp index a61ef37..333e9a3 100644 --- a/examples/poiseulle_particles_2d_gpu/init.hpp +++ b/examples/poiseulle_particles_2d_gpu/init.hpp @@ -57,7 +57,7 @@ kel::lbm::error_or<void> init( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -113,7 +113,7 @@ kel::lbm::error_or<void> init( ); { - kel::lbm::data<sch::Scalar<T>> radius_p; + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; radius_p.at({}).set(2); kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); diff --git a/examples/poiseulle_particles_2d_gpu/sim.cpp b/examples/poiseulle_particles_2d_gpu/sim.cpp index e852313..f3c001b 100644 --- a/examples/poiseulle_particles_2d_gpu/sim.cpp +++ b/examples/poiseulle_particles_2d_gpu/sim.cpp @@ -2,12 +2,13 @@ #include "init.hpp" #include "step.hpp" -#include <forstio/codec/args.hpp> /** * For deciding what parameters to use maybe? */ namespace args { +using namespace kel::lbm::sch; + using LbmArgsStruct = Struct< Member<UInt8,"use_slip">, @@ -46,9 +47,9 @@ kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); @@ -197,7 +198,7 @@ kel::lbm::error_or<void> k_main(int argc, char** argv){ kel::lbm::data<args::LbmArgs> args; { - auto eov = saw::parse_args(args,argc,argv); + auto eov = kel::lbm::parse_args(args,argc,argv); return eov; } auto& an = args.template get<"args">(); diff --git a/examples/poiseulle_particles_2d_hlbm_gpu/sim.cpp b/examples/poiseulle_particles_2d_hlbm_gpu/sim.cpp index 926086c..f96a022 100644 --- a/examples/poiseulle_particles_2d_hlbm_gpu/sim.cpp +++ b/examples/poiseulle_particles_2d_hlbm_gpu/sim.cpp @@ -45,7 +45,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> @@ -105,7 +105,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -280,9 +280,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/poiseulle_particles_2d_ibm_gpu/sim.cpp b/examples/poiseulle_particles_2d_ibm_gpu/sim.cpp index b051c4e..dd284dd 100644 --- a/examples/poiseulle_particles_2d_ibm_gpu/sim.cpp +++ b/examples/poiseulle_particles_2d_ibm_gpu/sim.cpp @@ -15,7 +15,7 @@ namespace lbm { constexpr uint64_t dim_y = 256ul; constexpr uint64_t dim_x = dim_y * 20ul; -// constexpr uint64_t particle_amount = 1ul; +constexpr uint64_t particle_amount = 1ul; namespace sch { @@ -45,7 +45,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force"> >; @@ -56,11 +56,8 @@ using MacroStruct = Struct< //>; template<typename T, typename Desc> -using ParticleSpheroidGroup = ParticleGroup< - T, - Desc::D, - sch::ParticleCollisionSpheroid<T,2.0f> ->; +using ParticleSpheroidGroup = + ParticleGroup<T,Desc::D,particle_amount,coll::Spheroid<T>>; } @@ -112,7 +109,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& force_f = macros.template get<"force">(); iterator<Desc::D>::apply( @@ -152,20 +149,22 @@ kel::lbm::error_or<void> setup_initial_conditions( // Particles { + // The radius is a per particle property now, and a lattice length. + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad_p; + rad_p.at({}).set(2.0); kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); auto& spheroid_grp = particles; - spheroid_grp = create_spheroid_particle_group<T,Desc::D,2.0f>( + spheroid_grp = create_spheroid_particle_group<T,Desc::D,particle_amount>( + rad_p, dense_p, {32u} ); { auto& p = spheroid_grp.template get<"particles">(); - p = {{{1u}}}; - iterator<1u>::apply( [&](auto& index){ auto& p_ind = p.at(index); @@ -180,7 +179,7 @@ kel::lbm::error_or<void> setup_initial_conditions( p_pos_old = p_pos; }, {}, - p.meta() + p.get_dims() ); } } @@ -217,13 +216,12 @@ kel::lbm::error_or<void> step( q.submit([&](acpp::sycl::handler& h){ h.parallel_for(acpp::sycl::range<1u>{1u}, [=](acpp::sycl::id<1u> idx){ - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& dense_f = macros.template get<"density">(); auto& ps = particles; auto& mask = ps.template get<"mask">(); auto& mask_step = ps.template get<"mask_step">().at({}); - auto& p_dense = ps.template get<"density">().at({}); auto& com = ps.template get<"center_of_mass">(); auto& parts = ps.template get<"particles">(); @@ -235,7 +233,8 @@ kel::lbm::error_or<void> step( /// In this case it's simple since I'm too lazy to do scaling and rotation /// Technically scale => rotate => translate /// Here it's only translate - auto& p_i_rb_pos = p_i_rb.template get<"position">(); + // Unwrapped: the IBM shift works in raw lattice offsets. + auto& p_i_rb_pos = p_i_rb.template get<"position">().handle(); iterator<Desc::D>::apply([&](const auto& index){ /// Calculate the shift from the mask @@ -371,9 +370,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/poiseulle_particles_2d_psm_gpu/sim.cpp b/examples/poiseulle_particles_2d_psm_gpu/sim.cpp index 3fd5579..122fd9f 100644 --- a/examples/poiseulle_particles_2d_psm_gpu/sim.cpp +++ b/examples/poiseulle_particles_2d_psm_gpu/sim.cpp @@ -46,7 +46,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> @@ -123,7 +123,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -320,9 +320,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/poiseulle_particles_channel_2d/poiseulle_particles_channel_2d.cpp b/examples/poiseulle_particles_channel_2d/poiseulle_particles_channel_2d.cpp index aadeb12..44d874d 100644 --- a/examples/poiseulle_particles_channel_2d/poiseulle_particles_channel_2d.cpp +++ b/examples/poiseulle_particles_channel_2d/poiseulle_particles_channel_2d.cpp @@ -67,7 +67,7 @@ using CellStruct = Struct< template<typename T, uint64_t D> using MacroStruct = Struct< - Member<Vector<T,D>, "momentum">, + Member<Vector<T,D>, "velocity">, Member<T, "pressure">, Member<UInt16, "particle">, Member<FixedArray<T,D>, "force"> @@ -368,7 +368,7 @@ void couple_particles_to_lattice( // Interpolate this from close U cells. // For now pick the closest U - auto& closest_u = macros.at(p_cell_pos).template get<"momentum">(); + auto& closest_u = macros.at(p_cell_pos).template get<"velocity">(); // auto p_shift = closest_u; // p_pos - p_pos_old @@ -720,7 +720,7 @@ kel::lbm::error_or<void> real_main(int argc, char** argv){ auto& macro_cell = macros.at(index); auto& rho = macro_cell.template get<"pressure">(); - auto& vel = macro_cell.template get<"momentum">(); + auto& vel = macro_cell.template get<"velocity">(); auto& force = macro_cell.template get<"force">(); auto& part_mask = macro_cell.template get<"particle">(); diff --git a/examples/schaefer_turek_durst_krause_rannbacher/common.hpp b/examples/schaefer_turek_durst_krause_rannbacher/common.hpp index cd5d944..74ef1b1 100644 --- a/examples/schaefer_turek_durst_krause_rannbacher/common.hpp +++ b/examples/schaefer_turek_durst_krause_rannbacher/common.hpp @@ -71,7 +71,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x*N, dim_y*N>; template<typename T, typename Desc, uint64_t N = 1u> using MacroStruct = Struct< - Member<VelChunk<T,Desc,N>, "momentum">, + Member<VelChunk<T,Desc,N>, "velocity">, Member<RhoChunk<T,N>, "density">, Member<ScalarChunk<T,Desc,N>, "porosity">, Member<VelChunk<T,Desc,N>, "force"> diff --git a/examples/schaefer_turek_durst_krause_rannbacher/init.hpp b/examples/schaefer_turek_durst_krause_rannbacher/init.hpp index c8e2cc9..b6db4ea 100644 --- a/examples/schaefer_turek_durst_krause_rannbacher/init.hpp +++ b/examples/schaefer_turek_durst_krause_rannbacher/init.hpp @@ -57,7 +57,7 @@ kel::lbm::error_or<void> init( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -94,8 +94,8 @@ kel::lbm::error_or<void> init( ); { - kel::lbm::data<sch::Scalar<T>> radius_p; - radius_p.at({}).set(conv.meter_si_to_lbm({{0.05}}).handle().get()); + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; + radius_p.at({}).set(conv.meter_si_to_lbm({0.05}).handle().get()); std::cout<<"RADIUS: "<<radius_p.at({}).get()<<std::endl; kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); @@ -107,8 +107,8 @@ kel::lbm::error_or<void> init( auto& prb = p.template get<"rigid_body">(); auto& p_pos = prb.template get<"position">(); auto& p_posold = prb.template get<"position_old">(); - p_pos.at({{0u}}) = conv.meter_si_to_lbm({{0.16}}).handle(); - p_pos.at({{1u}}) = conv.meter_si_to_lbm({{0.2}}).handle(); + p_pos.at({{0u}}) = conv.meter_si_to_lbm({0.16}).handle(); + p_pos.at({{1u}}) = conv.meter_si_to_lbm({0.2}).handle(); p_posold = p_pos; } diff --git a/examples/schaefer_turek_durst_krause_rannbacher/sim.cpp b/examples/schaefer_turek_durst_krause_rannbacher/sim.cpp index 2d389d1..5446639 100644 --- a/examples/schaefer_turek_durst_krause_rannbacher/sim.cpp +++ b/examples/schaefer_turek_durst_krause_rannbacher/sim.cpp @@ -2,12 +2,13 @@ #include "init.hpp" #include "step.hpp" -#include <forstio/codec/args.hpp> /** * For deciding what parameters to use maybe? */ namespace args { +using namespace kel::lbm::sch; + using LbmArgsStruct = Struct< Member<String, "coupling"> @@ -45,9 +46,9 @@ kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ converter<T> conv { // delta_x - {{0.41 / (dim_y*N)}}, + {0.41 / (dim_y*N)}, // delta_t - {{3e-3/(N*N)}} + {3e-3/(N*N)} }; print_lbm_meta<T,Desc>(conv,{1e-3},{0.3},{0.1}); @@ -203,7 +204,7 @@ kel::lbm::error_or<void> k_main(int argc, char** argv){ kel::lbm::data<args::LbmArgs> args; { - auto eov = saw::parse_args(args,argc,argv); + auto eov = kel::lbm::parse_args(args,argc,argv); if(eov.is_error()){ return eov; } diff --git a/examples/schaefer_turek_durst_krause_rannbacher/step.hpp b/examples/schaefer_turek_durst_krause_rannbacher/step.hpp index 080d27d..adb0740 100644 --- a/examples/schaefer_turek_durst_krause_rannbacher/step.hpp +++ b/examples/schaefer_turek_durst_krause_rannbacher/step.hpp @@ -73,8 +73,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -149,8 +149,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -225,8 +225,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; diff --git a/examples/settling_cubes_2d_ibm_gpu/sim.cpp b/examples/settling_cubes_2d_ibm_gpu/sim.cpp index 50e6ce5..9c41fa9 100644 --- a/examples/settling_cubes_2d_ibm_gpu/sim.cpp +++ b/examples/settling_cubes_2d_ibm_gpu/sim.cpp @@ -44,17 +44,14 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VectorChunk<T,Desc>, "force"> >; template<typename T, typename Desc> -using ParticleSpheroidGroup = ParticleGroup< - T, - Desc::D, - sch::ParticleCollisionSpheroid<T,2.0f> ->; +using ParticleSpheroidGroup = + ParticleGroup<T,Desc::D,particle_amount,coll::Spheroid<T>>; template<typename T, typename Desc> using ParticleGroups = Tuple< @@ -90,9 +87,9 @@ kel::lbm::error_or<void> setup_initial_conditions( {{1u,1u}} ); // - auto& df_f = fields.template gStarted hearing about similar cases not long after, many of which were successful on the part of the crooks. Scary stuff. et<"dfs_old">(); + auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); iterator<Desc::D>::apply( [&](auto& index){ @@ -110,12 +107,16 @@ kel::lbm::error_or<void> setup_initial_conditions( // Particles { + // Radius is per particle now, and a lattice length. + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad_p; + rad_p.at({}).set(2.0); kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); // auto& spheroid_group = particles.template get<0u>(); auto& spheroid_group = particles; - spheroid_group = create_spheroid_particle_group<T,Desc::D,2.0f>( + spheroid_group = create_spheroid_particle_group<T,Desc::D,particle_amount>( + rad_p, dense_p, {64u} ); @@ -123,8 +124,6 @@ kel::lbm::error_or<void> setup_initial_conditions( { auto& p = spheroid_group.template get<"particles">(); - p = {{{16u}}}; - iterator<1u>::apply( [&](auto& index){ // Set Pos here? @@ -139,7 +138,7 @@ kel::lbm::error_or<void> setup_initial_conditions( p_pos_old = p_pos; }, {}, - p.meta() + p.get_dims() ); } } @@ -162,7 +161,7 @@ kel::lbm::error_or<void> step( auto& parts = p_group.template get<"particles">(); auto& p_mask = p_group.template get<"mask">(); - auto& vels = macros.template get<"momentum">(); + auto& vels = macros.template get<"velocity">(); auto& forces = macros.template get<"force">(); auto p_meta = parts.meta(); @@ -189,13 +188,13 @@ kel::lbm::error_or<void> step( auto& p = parts.at(index); auto& p_rb = p.template get<"rigid_body">(); - kel::lbm::data<sch::Scalar<T>> delta_t; + kel::lbm::data<sch::LbmSecond<sch::Scalar<T>>> delta_t; // verlet takes a lattice time delta_t.at({}).set(1.0f); - auto& p_pos = p_rb.template get<"position">(); + auto& p_pos = p_rb.template get<"position">().handle(); // raw: IBM works in lattice offsets auto& p_rot = p_rb.template get<"rotation">(); - auto& p_acc = p_rb.template get<"acceleration">(); + auto& p_acc = p_rb.template get<"acceleration">().handle(); // raw: the force below is not divided by mass yet (see TODO) iterator<Desc::D>::apply( [&](auto& m_ind){ @@ -227,12 +226,12 @@ kel::lbm::error_or<void> step( // this is our relative position to the particle auto rel_cell_to_part_pos = p_cell_pos_vec.template cast_to<T>() - p_pos; - auto p_vel = (p_pos - p_rb.template get<"position_old">()) * delta_t; + auto p_vel = (p_pos - p_rb.template get<"position_old">().handle()) * delta_t.handle(); auto u_solid = p_vel + kel::lbm::math::cross(p_rot,rel_cell_to_part_pos); // Force - auto force = (u_solid - u_fluid) / delta_t; + auto force = (u_solid - u_fluid) / delta_t.handle(); // TODO HERE ATOMIC! !!!! forces.at(p_cell_pos) = forces.at(p_cell_pos) + force; @@ -295,13 +294,13 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ using dfi = df_info<T,Desc>; - auto eo_lbm_env = lbm_directory(); - if(eo_lbm_env.is_error()){ - return std::move(eo_lbm_env.get_error()); + auto eo_lbm_dir = output_directory(); + if(eo_lbm_dir.is_error()){ + return std::move(eo_lbm_dir.get_error()); } - auto& lbm_env = eo_lbm_env.get_value(); + auto& lbm_dir = eo_lbm_dir.get_value(); - auto out_dir = lbm_env.data_dir / "settling_cubes_2d_ibm_gpu"; + auto out_dir = lbm_dir / "settling_cubes_2d_ibm_gpu"; { std::error_code ec; @@ -313,9 +312,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.05},{0.01},{0.4 * dim_y}); diff --git a/examples/settling_spheres_2d_hlbm_gpu/sim.cpp b/examples/settling_spheres_2d_hlbm_gpu/sim.cpp index 26c2ed8..201a47c 100644 --- a/examples/settling_spheres_2d_hlbm_gpu/sim.cpp +++ b/examples/settling_spheres_2d_hlbm_gpu/sim.cpp @@ -33,8 +33,8 @@ using ChunkStruct = Struct< Member<InfoChunk, "info">, Member<DfChunk<T,Desc>, "dfs">, Member<DfChunk<T,Desc>, "dfs_old">, - Member<VectorChunk<T,Desc>, "p_N">, - Member<ScalarChunk<T,Desc>, "p_D"> + Member<VectorChunk<T,Desc>, "particle_N">, + Member<ScalarChunk<T,Desc>, "particle_D"> >; template<typename T, typename Desc> @@ -48,8 +48,10 @@ using PorChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, + // HlbmOneParticleGather reads this one. + Member<VelChunk<T,Desc>, "force">, Member<PorChunk<T>, "porosity"> >; @@ -88,9 +90,9 @@ kel::lbm::error_or<void> setup_initial_conditions( {{1u,1u}} ); // - auto& df_f = fields.template gStarted hearing about similar cases not long after, many of which were successful on the part of the crooks. Scary stuff. et<"dfs_old">(); + auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); iterator<Desc::D>::apply( [&](auto& index){ @@ -108,12 +110,16 @@ kel::lbm::error_or<void> setup_initial_conditions( // Particles { + // Radius is per particle now, and a lattice length. + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad_p; + rad_p.at({}).set(2.0); kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); // auto& spheroid_group = particles.template get<0u>(); - auto& spheroid_group = particles; + auto& spheroid_group = p_group; - spheroid_group = create_spheroid_particle_group<T,Desc::D,2.0f>( + spheroid_group = create_spheroid_particle_group<T,Desc::D,particle_amount>( + rad_p, dense_p, {64u} ); @@ -121,9 +127,6 @@ kel::lbm::error_or<void> setup_initial_conditions( { auto& p = spheroid_group.template get<"particles">(); - /// 16 if I remember correctly? - p = {{{particle_amount}}}; - iterator<1u>::apply( [&](auto& index){ // Set Pos here? @@ -138,13 +141,13 @@ kel::lbm::error_or<void> setup_initial_conditions( p.at({{0u}}).set( 0.2 + 0.6 / (x-1u) ); p.at({{1u}}).set( 0.8 + y * 0.1 ); - p_pos = p; + p_pos.handle() = p; auto& p_pos_old = p_rb.template get<"position_old">(); p_pos_old = p_pos; }, {}, - p.meta() + p.get_dims() ); } } @@ -167,7 +170,7 @@ kel::lbm::error_or<void> step( auto& parts = p_group.template get<"particles">(); auto& p_mask = p_group.template get<"mask">(); - auto& vels = macros.template get<"momentum">(); + auto& vels = macros.template get<"velocity">(); auto& forces = macros.template get<"force">(); auto p_meta = parts.meta(); @@ -196,9 +199,9 @@ kel::lbm::error_or<void> step( auto& p_rot = pirb.template get<"rotation">(); auto& p_acc = pirb.template get<"acceleration">(); - kel::lbm::data<sch::Scalar<T>> delta_t; + kel::lbm::data<sch::LbmSecond<sch::Scalar<T>>> delta_t; delta_t.at({}).set(1.0f); - verlet_step_lambda<T,Desc::D>(p,delta_t); + verlet_step_lambda<T,Desc::D>(pi,delta_t); }); }).wait(); @@ -249,13 +252,13 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ using dfi = df_info<T,Desc>; - auto eo_lbm_env = lbm_directory(); - if(eo_lbm_env.is_error()){ - return std::move(eo_lbm_env.get_error()); + auto eo_lbm_dir = output_directory(); + if(eo_lbm_dir.is_error()){ + return std::move(eo_lbm_dir.get_error()); } - auto& lbm_env = eo_lbm_env.get_value(); + auto& lbm_dir = eo_lbm_dir.get_value(); - auto out_dir = lbm_env.data_dir / "settling_spheres_2d_hlbm_gpu"; + auto out_dir = lbm_dir / "settling_spheres_2d_hlbm_gpu"; { std::error_code ec; @@ -267,9 +270,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.05},{0.01},{0.4 * dim_y}); diff --git a/examples/stdkr_2d_bgk/common.hpp b/examples/stdkr_2d_bgk/common.hpp index cac01fd..64fe124 100644 --- a/examples/stdkr_2d_bgk/common.hpp +++ b/examples/stdkr_2d_bgk/common.hpp @@ -57,7 +57,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density"> >; diff --git a/examples/stdkr_2d_bgk/init.hpp b/examples/stdkr_2d_bgk/init.hpp index 10edd9c..d37c8de 100644 --- a/examples/stdkr_2d_bgk/init.hpp +++ b/examples/stdkr_2d_bgk/init.hpp @@ -55,7 +55,7 @@ kel::lbm::error_or<void> init( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); iterator<Desc::D>::apply( [&](auto& index){ @@ -93,13 +93,13 @@ kel::lbm::error_or<void> init( [&](const auto& index){ auto& info = info_f.at(index); kel::lbm::data<sch::Scalar<T>> radius_p; - radius_p.at({}).set(conv.meter_si_to_lbm({{0.05}}).handle().get()); + radius_p.at({}).set(conv.meter_si_to_lbm({0.05}).handle().get()); kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); kel::lbm::data<sch::Vector<T,Desc::D>> middle,ind_vec; - middle.at({{0u}}) = conv.meter_si_to_lbm({{0.16}}).handle(); - middle.at({{1u}}) = conv.meter_si_to_lbm({{0.2}}).handle(); + middle.at({{0u}}) = conv.meter_si_to_lbm({0.16}).handle(); + middle.at({{1u}}) = conv.meter_si_to_lbm({0.2}).handle(); for(uint64_t i{0u}; i < Desc::D; ++i){ ind_vec.at({{i}}) = index.at({i}).template cast_to<T>(); diff --git a/examples/stdkr_2d_bgk/sim.cpp b/examples/stdkr_2d_bgk/sim.cpp index 9b9b5a7..b388e3c 100644 --- a/examples/stdkr_2d_bgk/sim.cpp +++ b/examples/stdkr_2d_bgk/sim.cpp @@ -45,9 +45,9 @@ kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ converter<T> conv { // delta_x - {{0.41 / dim_y}}, + {0.41 / dim_y}, // delta_t - {{1.067e-4}} + {1.067e-4} }; print_lbm_meta<T,Desc>(conv,{1e-3},{1e-4},{0.5 * dim_y}); diff --git a/examples/stdkr_2d_bgk/step.hpp b/examples/stdkr_2d_bgk/step.hpp index 67f8486..d7d4fd4 100644 --- a/examples/stdkr_2d_bgk/step.hpp +++ b/examples/stdkr_2d_bgk/step.hpp @@ -48,7 +48,7 @@ kel::lbm::error_or<void> step( kel::lbm::data<sch::Vector<T,Desc::D>> vel; { auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}}).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; diff --git a/examples/stokes_drag_particle_2d_bgk_gpu/sim.cpp b/examples/stokes_drag_particle_2d_bgk_gpu/sim.cpp index ffa921f..38d059a 100644 --- a/examples/stokes_drag_particle_2d_bgk_gpu/sim.cpp +++ b/examples/stokes_drag_particle_2d_bgk_gpu/sim.cpp @@ -48,7 +48,7 @@ using ForceChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density"> >; @@ -121,7 +121,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); iterator<Desc::D>::apply( [&](auto& index){ @@ -292,9 +292,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/stokes_drag_particle_2d_fplbm_gpu/sim.cpp b/examples/stokes_drag_particle_2d_fplbm_gpu/sim.cpp index 90a73cd..37551bc 100644 --- a/examples/stokes_drag_particle_2d_fplbm_gpu/sim.cpp +++ b/examples/stokes_drag_particle_2d_fplbm_gpu/sim.cpp @@ -48,7 +48,7 @@ using ForceChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> @@ -124,7 +124,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -299,9 +299,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/stokes_drag_particle_2d_hlbm_gpu/sim.cpp b/examples/stokes_drag_particle_2d_hlbm_gpu/sim.cpp index 8117b19..1d20b80 100644 --- a/examples/stokes_drag_particle_2d_hlbm_gpu/sim.cpp +++ b/examples/stokes_drag_particle_2d_hlbm_gpu/sim.cpp @@ -45,7 +45,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> @@ -121,7 +121,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -298,9 +298,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/stokes_drag_particle_2d_psm_gpu/sim.cpp b/examples/stokes_drag_particle_2d_psm_gpu/sim.cpp index 49a6128..fc9fdc1 100644 --- a/examples/stokes_drag_particle_2d_psm_gpu/sim.cpp +++ b/examples/stokes_drag_particle_2d_psm_gpu/sim.cpp @@ -46,7 +46,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<VelChunk<T,Desc>, "force">, Member<ScalarChunk<T,Desc>, "porosity"> @@ -123,7 +123,7 @@ kel::lbm::error_or<void> setup_initial_conditions( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -281,9 +281,9 @@ kel::lbm::error_or<void> lbm_main(int argc, char** argv){ converter<T> conv { // delta_x - {{1.0}}, + {1.0}, // delta_t - {{1.0}} + {1.0} }; print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); diff --git a/examples/throat_clog_2d/common.hpp b/examples/throat_clog_2d/common.hpp index 84baeb3..8fb9840 100644 --- a/examples/throat_clog_2d/common.hpp +++ b/examples/throat_clog_2d/common.hpp @@ -62,7 +62,7 @@ using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; template<typename T, typename Desc> using MacroStruct = Struct< - Member<VelChunk<T,Desc>, "momentum">, + Member<VelChunk<T,Desc>, "velocity">, Member<RhoChunk<T>, "density">, Member<ScalarChunk<T,Desc>, "porosity">, Member<VelChunk<T,Desc>, "force"> diff --git a/examples/throat_clog_2d/init.hpp b/examples/throat_clog_2d/init.hpp index e03bd5e..fab51b3 100644 --- a/examples/throat_clog_2d/init.hpp +++ b/examples/throat_clog_2d/init.hpp @@ -120,7 +120,7 @@ kel::lbm::error_or<void> init( // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); - auto& vel_f = macros.template get<"momentum">(); + auto& vel_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); iterator<Desc::D>::apply( @@ -157,8 +157,8 @@ kel::lbm::error_or<void> init( ); { - kel::lbm::data<sch::Scalar<T>> radius_p; - radius_p.at({}).set(conv.meter_si_to_lbm({{0.05}}).handle().get()); + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; + radius_p.at({}).set(conv.meter_si_to_lbm({0.05}).handle().get()); std::cout<<"RADIUS: "<<radius_p.at({}).get()<<std::endl; kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); @@ -175,8 +175,8 @@ kel::lbm::error_or<void> init( const auto radius = radius_p.at({}).get(); const auto pitch = radius * 2.4; - const auto x0 = conv.meter_si_to_lbm({{0.10}}).handle().get(); - const auto y_mid = conv.meter_si_to_lbm({{0.25}}).handle().get(); + const auto x0 = conv.meter_si_to_lbm({0.10}).handle().get(); + const auto y_mid = conv.meter_si_to_lbm({0.25}).handle().get(); constexpr uint64_t per_column = 4ul; diff --git a/examples/throat_clog_2d/sim.cpp b/examples/throat_clog_2d/sim.cpp index 89741ca..a25c1a3 100644 --- a/examples/throat_clog_2d/sim.cpp +++ b/examples/throat_clog_2d/sim.cpp @@ -33,9 +33,9 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ converter<T> conv { // delta_x - {{0.64 / dim_y}}, + {0.64 / dim_y}, // delta_t - {{1.067e-2}} + {1.067e-2} }; print_lbm_meta<T,Desc>(conv,{1e-3},{1e-4},{0.2 * dim_y}); diff --git a/examples/throat_clog_2d/step.hpp b/examples/throat_clog_2d/step.hpp index 5bb889e..1605cf0 100644 --- a/examples/throat_clog_2d/step.hpp +++ b/examples/throat_clog_2d/step.hpp @@ -83,8 +83,8 @@ kel::lbm::error_or<void> step( }()); } { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((0.1 / 0.64) * y_si.get() - (0.1 / (0.64*0.64)) * y_si.get() * y_si.get())}}).handle() * scale; + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((0.1 / 0.64) * y_si.get() - (0.1 / (0.64*0.64)) * y_si.get() * y_si.get())}).handle() * scale; vel.at({{1u}}) = 0.0f; } return vel; @@ -178,8 +178,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((0.01 / 0.64) * y_si.get() - (0.01 / (0.64*0.64)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((0.01 / 0.64) * y_si.get() - (0.01 / (0.64*0.64)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -254,8 +254,8 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({{static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.64) * y_si.get() - (1.2 / (0.64*0.64)) * y_si.get() * y_si.get())}}).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.64) * y_si.get() - (1.2 / (0.64*0.64)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; diff --git a/examples/vawt_3d_hlbm_gpu/.nix/derivation.nix b/examples/vawt_3d_hlbm_gpu/.nix/derivation.nix new file mode 100644 index 0000000..8d64547 --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/.nix/derivation.nix @@ -0,0 +1,41 @@ +{ lib +, stdenv +, scons +, clang-tools +, forstio +, python3 +, pname +, version +, adaptive-cpp +, kel +}: + +stdenv.mkDerivation { + pname = pname + "-examples-" + "vawt_3d_hlbm_gpu"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + python3 + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + forstio.codec-unit + forstio.io + forstio.remote + forstio.remote-filesystem + forstio.codec-json + adaptive-cpp + kel.lbm.core + kel.lbm.sycl + ]; + + preferLocalBuild = true; + + outputs = [ "out" "dev" ]; +} diff --git a/examples/vawt_3d_hlbm_gpu/SConscript b/examples/vawt_3d_hlbm_gpu/SConscript new file mode 100644 index 0000000..aecf80a --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/SConscript @@ -0,0 +1,34 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +examples_env = env.Clone(); +examples_env['CXX'] = 'syclcc-clang'; +examples_env['CXXFLAGS'] += ['-O3']; + +examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += examples_env.sources; +env.headers += examples_env.headers; + +# Cavity2D +examples_objects = []; +examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); +examples_env.poiseulle_2d_gpu = examples_env.Program('#bin/vawt_3d_hlbm_gpu', [examples_objects]); + +# Set Alias +env.examples = [ + examples_env.poiseulle_2d_gpu +]; +env.Alias('examples', env.examples); +env.targets += ['examples']; +env.Install('$prefix/bin/', env.examples); diff --git a/examples/vawt_3d_hlbm_gpu/SConstruct b/examples/vawt_3d_hlbm_gpu/SConstruct new file mode 100644 index 0000000..0611b67 --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/SConstruct @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env_vars.Add('build_examples', + help='If examples should be built', + default="true" +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=[ + '-std=c++20', + '-g', + '-Wall', + '-Wextra' + ], + LIBS=[ + 'forstio-core', + 'forstio-async', + 'forstio-io' + ] +); +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/examples/vawt_3d_hlbm_gpu/sim.cpp b/examples/vawt_3d_hlbm_gpu/sim.cpp new file mode 100644 index 0000000..28d404b --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/sim.cpp @@ -0,0 +1,656 @@ +#include <kel/lbm/sycl/lbm.hpp> +#include <kel/lbm/lbm.hpp> +#include <kel/lbm/particle.hpp> + +#include <fstream> + +#include <forstio/io/io.hpp> +#include <forstio/remote/filesystem/easy.hpp> +#include <forstio/codec/json/json.hpp> +#include <forstio/codec/simple.hpp> + +namespace kel { +namespace lbm { + +// Deliberately tiny: 32 x 32 x 32 = 32768 cells, so a full run finishes in +// seconds. Square in y/z so the channel cross section is symmetric about the +// particle, which makes a stray transverse force obvious. +constexpr uint64_t dim_x = 32ul; +constexpr uint64_t dim_y = 32ul; +constexpr uint64_t dim_z = 32ul; + +constexpr uint64_t particle_amount = 1ul; + +/** + * The particle lives in lattice space, exactly like the fluid: the radius is a + * number of cells, the density is relative to the fluid's rho = 1, and the + * mass, inertia, force and torque that follow are all lattice quantities. The + * converter handles the fluid's SI <-> LBM bookkeeping and is deliberately not + * applied to any of this -- so every column of particle.csv is in cells, + * steps, and lattice mass. + */ +constexpr double particle_radius = 0.15 * static_cast<double>(dim_y); +constexpr double particle_density = 1.0; + +namespace sch { + +/** + * The spin axis is an arbitrary fixed vector, not a lattice direction. It is + * normalised by the integrator, so these three need not be a unit vector. + * + * Defaults to +y: flow along x, axis vertical and perpendicular to it, blades + * sweeping the x-z plane -- the vertical-axis turbine arrangement. + */ +using LbmArgs = Args< + Struct< + Member<String, "name">, + Member<UInt64, "time_steps">, + Member<UInt64, "sub_steps">, + Member<Float64, "axis_x">, + Member<Float64, "axis_y">, + Member<Float64, "axis_z"> + >, + Tuple<> +>; + +using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using DfChunk = Chunk<FixedArray<T,Desc::Q>, 1u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using ScalarChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using VectorChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; + +template<typename T, typename Desc> +using ChunkStruct = Struct< + Member<InfoChunk, "info">, + Member<DfChunk<T,Desc>, "dfs">, + Member<DfChunk<T,Desc>, "dfs_old">, + Member<VectorChunk<T,Desc>, "particle_N">, + Member<ScalarChunk<T,Desc>, "particle_D"> +>; + +template<typename T, typename Desc> +using MomChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; + +template<typename T> +using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; + +/** + * "force" is not optional here: HlbmOneParticleGather reads it, so the 3D + * macro struct needs it the moment a particle is carried. + */ +template<typename T, typename Desc> +using MacroStruct = Struct< + Member<MomChunk<T,Desc>, "velocity">, + Member<RhoChunk<T>, "density">, + Member<MomChunk<T,Desc>, "force">, + Member<ScalarChunk<T,Desc>, "porosity"> +>; + +/** + * particle_aabb is specialised only on ParticleGroup, so a bare + * FixedArray<Particle<...>> hits its always_false primary template. + */ +template<typename T, typename Desc> +using ParticleSpheroidGroup = ParticleGroup<T,Desc::D,particle_amount,coll::Spheroid<T>>; +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> setup_initial_conditions( + converter<T>& conv, + kel::lbm::data<sch::ChunkStruct<T,Desc>>& fields, + kel::lbm::data<sch::MacroStruct<T,Desc>>& macros, + kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>>& particles +){ + // Unit conversion is not exercised in this reduced case. + (void) conv; + + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u,1u}} + ); + + // Inflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u,0u}}, + {{1u,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + + // Outflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u,0u}}, + {{dim_x,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& mom_f = macros.template get<"velocity">(); + auto& por_f = macros.template get<"porosity">(); + auto& force_f = macros.template get<"force">(); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + por_f.at(index).at({}) = {1}; + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + auto eq = equilibrium<T,Desc>(rho,mom); + + df = eq; + + auto& force = force_f.at(index); + for(uint64_t i{0u}; i < Desc::D; ++i){ + force.at({{i}}).set(0); + } + }, + {},// 0-index + df_f.get_dims() + ); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + mom.at({{0u}}) = 0.01; + auto eq = equilibrium<T,Desc>(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u,1u}} + ); + + kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> rad; + rad.at({}).set(particle_radius); + kel::lbm::data<sch::Scalar<T>> dense; + dense.at({}).set(particle_density); + + // Mask resolution 16 in 3D is 16^3 = 4096 voxels. The analytic spheroid + // porosity is what HLBM actually calls, but the mask is still filled. + particles = create_spheroid_particle_group<T,Desc::D,particle_amount>(rad,dense,{16u}); + { + auto& rb = particles.template get<"particles">().at({0u}).template get<"rigid_body">(); + auto& rbp = rb.template get<"position">(); + rbp.at({{0u}}) = 0.3 * dim_x; + rbp.at({{1u}}) = 0.5 * dim_y; + rbp.at({{2u}}) = 0.5 * dim_z; + + // position_old == position means the body starts at rest. It also + // makes a pinned body exact later: 2x - x_old + 0 == x. + rb.template get<"position_old">() = rbp; + + kel::lbm::data<sch::Vector<T,Desc::D>> zero; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero.at({{i}}).set(0); + } + // Acceleration is unit typed now, so its zero has to be too. + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> zero_acc; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero_acc.at({{i}}).set(0); + } + rb.template get<"acceleration">() = zero_acc; + + // In 3D these are Vector<T,3> (a rotation vector), not a scalar angle. + rb.template get<"rotation">() = zero; + rb.template get<"rotation_old">() = zero; + rb.template get<"angular_acceleration">() = zero; + } + + return kel::lbm::make_void(); +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> step( + converter<T>& conv, + kel::lbm::data<sch::Ptr<sch::ChunkStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& fields, + kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, + kel::lbm::data<sch::Ptr<sch::ParticleSpheroidGroup<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& particles, + kel::lbm::data<sch::UInt64> t_i, + kel::lbm::data<sch::UInt64> sub_steps, + const kel::lbm::data<sch::Vector<T,3u>>& spin_axis, + device& dev +){ + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); + + // Porosity back to 1 (all fluid) before the particle restamps itself. + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::HlbmReset,encode::Sycl<kel::lbm::enc::Native>> hlbm_reset; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + hlbm_reset.apply(fields,macros,index,t_i); + }); + }).wait(); + + /** + * One work item per particle. The stamp has to come after the integrate: + * it writes the post-integration surface velocity back onto the lattice. + */ + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::ParticleForceReset,encode::Sycl<kel::lbm::enc::Native>> p_reset; + component<T,Desc,cmpt::HlbmOneParticleGather,encode::Sycl<kel::lbm::enc::Native>> p_gather; + component<T,Desc,cmpt::HlbmOneParticleStamp,encode::Sycl<kel::lbm::enc::Native>> p_stamp; + + /** + * The rotor replaces the free-body integrator. No contact pass: the + * body is pinned, so it cannot reach a wall or another particle. + */ + component<T,Desc,cmpt::ParticleAxisIntegrate,encode::Sycl<kel::lbm::enc::Native>> p_integrate{spin_axis}; + + h.parallel_for(acpp::sycl::range<1u>{particle_amount}, [=](acpp::sycl::id<1u> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,1u>> index; + for(uint64_t i = 0u; i < 1u; ++i){ + index.at({{i}}).set(idx[i]); + } + + p_reset.apply(particles,index,t_i); + p_gather.apply(fields,macros,particles,index,t_i,sub_steps); + p_integrate.apply(particles,index,t_i,sub_steps); + p_stamp.apply(fields,macros,particles,index,t_i,sub_steps); + }); + }).wait(); + + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + /** + * Hlbm rather than BGK: it is the collision that consumes the + * porosity and particle_N/particle_D the stamp pass writes. + */ + component<T,Desc,cmpt::Hlbm, encode::Sycl<kel::lbm::enc::Native>> collision{conv.template kinematic_viscosity_si_to_tau<Desc>({0.1})}; + component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; + + component<T,Desc,cmpt::ZouHeHorizontal<true>,encode::Sycl<kel::lbm::enc::Native>> flow_in{ + [&](){ + uint64_t target_t_i = 256u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.001 / target_t_i) * t_i.get(); + } + return 1.001; + }() + }; + component<T,Desc,cmpt::ZouHeHorizontal<false>,encode::Sycl<kel::lbm::enc::Native>> flow_out{1.0}; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: + break; + case 1u: + bb.apply(fields,index,t_i); + break; + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 3u: + flow_in.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: + break; + } + }); + }).wait(); + + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::Stream,encode::Sycl<kel::lbm::enc::Native>> stream; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(fields,index,t_i); + } + }); + }).wait(); + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ + + return kel::lbm::make_void(); +} +} +} + +template<typename T, typename Desc> +kel::lbm::error_or<void> lbm_main(int argc, char** argv){ + using namespace kel::lbm; + + kel::lbm::data<sch::LbmArgs> args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("vawt_3d_hlbm_gpu"); + auto& steps_arg = args_n.template get<"time_steps">(); + steps_arg.set(256u); + auto& sub_arg = args_n.template get<"sub_steps">(); + sub_arg.set(16u); + + // +y by default: flow along x, axis perpendicular to it. + args_n.template get<"axis_x">().set(0.0); + args_n.template get<"axis_y">().set(1.0); + args_n.template get<"axis_z">().set(0.0); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); + } + const auto& lbm_env = eo_lbm_env.get_value(); + + auto& out_dir = lbm_env.name_dir; + + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not create output directory"); + } + } + + converter<T> conv { + // delta_x + {1.0}, + // delta_t + {1.0} + }; + + // Characteristic length is the body diameter -- the scale that sets Re here. + print_lbm_meta<T,Desc>(conv,{0.1},{0.01},{2.0 * particle_radius}); + + auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); + auto lbm_macro_data_ptr = kel::lbm::heap<kel::lbm::data<sch::MacroStruct<T,Desc>>>(); + auto lbm_particle_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>>>(); + + std::cout<<"Estimated Bytes: "<<memory_estimate<sch::ChunkStruct<T,Desc>,sch::MacroStruct<T,Desc>>().get()<<std::endl; + + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error<kel::lbm::err::critical>( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; + + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); + + device dev; + + auto& sycl_q = dev.get_handle(); + + sycl_q.wait(); + { + auto eov = setup_initial_conditions<T,Desc>(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + kel::lbm::data<sch::ChunkStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_data{sycl_q}; + kel::lbm::data<sch::MacroStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_macro_data{sycl_q}; + kel::lbm::data<sch::ParticleSpheroidGroup<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_particle_data{sycl_q}; + sycl_q.wait(); + + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + auto lsdp_view = make_view(lbm_sycl_particle_data); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + kel::lbm::data<sch::UInt64> time_steps = args.template get<"args">().template get<"time_steps">(); + kel::lbm::data<sch::UInt64> sub_steps = args.template get<"args">().template get<"sub_steps">(); + + kel::lbm::data<sch::Vector<T,3u>> spin_axis; + { + auto& args_n = args.template get<"args">(); + spin_axis.at({{0u}}).set(args_n.template get<"axis_x">().get()); + spin_axis.at({{1u}}).set(args_n.template get<"axis_y">().get()); + spin_axis.at({{2u}}).set(args_n.template get<"axis_z">().get()); + } + + // Unit axis, for projecting the rotation and torque onto it below. + auto axis_hat = kel::lbm::math::normalize(spin_axis); + + /** + * Particle time series. write_vtk_file only accepts a struct of chunks, so + * a particle group cannot go through it -- a CSV beside the VTK output is + * the simpler answer, and it is what you actually plot. + * + * Velocity and angular velocity are not stored on the body: the scheme is + * position-Verlet, so both are recovered as (now - previous) * steps. + */ + std::ofstream particle_csv{out_dir / "particle.csv"}; + if(not particle_csv.is_open()){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not open particle.csv"); + } + /** + * theta/omega/alpha/tau are the rotation, angular velocity, angular + * acceleration and torque projected onto the spin axis -- the rotor + * numbers. drag is the streamwise force. The raw vectors follow so a + * stray off-axis component is still visible rather than hidden by the + * projection. + */ + particle_csv + <<"step,theta,omega,alpha,tau,drag," + <<"x,y,z,vx,vy,vz,fx,fy,fz," + <<"rot_x,rot_y,rot_z,omega_x,omega_y,omega_z," + <<"alpha_x,alpha_y,alpha_z,tau_x,tau_y,tau_z\n"; + + auto write_particle_row = [&](uint64_t step_i){ + auto& rb = lbm_particle_data_ptr->template get<"particles">() + .at({0u}).template get<"rigid_body">(); + + auto& pos = rb.template get<"position">(); + auto& pos_old = rb.template get<"position_old">(); + auto& force = rb.template get<"force">(); + auto& rot = rb.template get<"rotation">(); + auto& rot_old = rb.template get<"rotation_old">(); + auto& ang_acc = rb.template get<"angular_acceleration">(); + auto& torque = rb.template get<"torque">(); + + const auto steps = static_cast<double>(particle_sub_steps(sub_steps)); + + // Project onto the spin axis. + double theta = 0.0, omega = 0.0, alpha = 0.0, tau = 0.0; + for(uint64_t k{0u}; k < 3u; ++k){ + const double a = axis_hat.at({{k}}).get(); + theta += rot.at({{k}}).get() * a; + omega += (rot.at({{k}}).get() - rot_old.at({{k}}).get()) * steps * a; + alpha += ang_acc.at({{k}}).get() * a; + tau += torque.at({{k}}).get() * a; + } + + particle_csv + <<step_i + <<","<<theta + <<","<<omega + <<","<<alpha + <<","<<tau + <<","<<force.at({{0u}}).get(); + + particle_csv<<""; + for(uint64_t k{0u}; k < Desc::D; ++k){ + particle_csv<<","<<pos.at({{k}}).get(); + } + for(uint64_t k{0u}; k < Desc::D; ++k){ + particle_csv<<","<<(pos.at({{k}}).get() - pos_old.at({{k}}).get()) * steps; + } + for(uint64_t k{0u}; k < Desc::D; ++k){ + particle_csv<<","<<force.at({{k}}).get(); + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<rot.at({{k}}).get(); + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<(rot.at({{k}}).get() - rot_old.at({{k}}).get()) * steps; + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<ang_acc.at({{k}}).get(); + } + for(uint64_t k{0u}; k < 3u; ++k){ + particle_csv<<","<<torque.at({{k}}).get(); + } + particle_csv<<"\n"; + }; + + for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ + { + { + auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + { + auto eov = step<T,Desc>(conv,lsd_view,lsdm_view,lsdp_view,i,sub_steps,spin_axis,dev); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_particle_data,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + write_particle_row(i.get()); + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + wait.poll(); + if(print_status){ + std::cout<<"Status: "<<i.get()<<" of "<<time_steps.get()<<" - "<<(i.template cast_to<sch::Float64>().get() * 100 / time_steps.get())<<"%"<<std::endl; + print_status = false; + } + } + sycl_q.wait(); + { + auto eov = write_vtk_file(out_dir,"t",time_steps.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + /* + iterator<Desc::D>::apply( + [&](auto& index){ + std::cout<<index.at({0u}).get()<<" "<<index.at({1u}).get()<<" "<<lbm_data.get<"info">().at(index).template cast_to<sch::UInt16>().get()<<"\n"; + }, + {{0u,0u}}, + {{dim_x, dim_y}} + ); + */ + + sycl_q.wait(); + return kel::lbm::make_void(); +} + +using FloatT = kel::lbm::sch::Float32; + +int main(int argc, char** argv){ + auto eov = lbm_main<FloatT,kel::lbm::sch::D3Q27>(argc, argv); + if(eov.is_error()){ + auto& err = eov.get_error(); + std::cerr<<"[Error] "<<err.get_category(); + auto err_msg = err.get_message(); + if(err_msg.size() > 0u){ + std::cerr<<" - "<<err_msg; + } + std::cerr<<std::endl; + return err.get_id(); + } + return 0; +} |
