From 0c0087ef9482634273ec1a23d3a915754f94926d Mon Sep 17 00:00:00 2001 From: "Claudius \"keldu\" Holeksa" Date: Wed, 11 Nov 2026 01:16:44 +0100 Subject: Snapshot. Migration to Lbm Unit --- examples/vawt_3d_hlbm_gpu/.nix/derivation.nix | 41 ++ examples/vawt_3d_hlbm_gpu/SConscript | 34 ++ examples/vawt_3d_hlbm_gpu/SConstruct | 81 ++++ examples/vawt_3d_hlbm_gpu/sim.cpp | 656 ++++++++++++++++++++++++++ 4 files changed, 812 insertions(+) create mode 100644 examples/vawt_3d_hlbm_gpu/.nix/derivation.nix create mode 100644 examples/vawt_3d_hlbm_gpu/SConscript create mode 100644 examples/vawt_3d_hlbm_gpu/SConstruct create mode 100644 examples/vawt_3d_hlbm_gpu/sim.cpp (limited to 'examples/vawt_3d_hlbm_gpu') diff --git a/examples/vawt_3d_hlbm_gpu/.nix/derivation.nix b/examples/vawt_3d_hlbm_gpu/.nix/derivation.nix new file mode 100644 index 0000000..8d64547 --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/.nix/derivation.nix @@ -0,0 +1,41 @@ +{ lib +, stdenv +, scons +, clang-tools +, forstio +, python3 +, pname +, version +, adaptive-cpp +, kel +}: + +stdenv.mkDerivation { + pname = pname + "-examples-" + "vawt_3d_hlbm_gpu"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + python3 + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + forstio.codec-unit + forstio.io + forstio.remote + forstio.remote-filesystem + forstio.codec-json + adaptive-cpp + kel.lbm.core + kel.lbm.sycl + ]; + + preferLocalBuild = true; + + outputs = [ "out" "dev" ]; +} diff --git a/examples/vawt_3d_hlbm_gpu/SConscript b/examples/vawt_3d_hlbm_gpu/SConscript new file mode 100644 index 0000000..aecf80a --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/SConscript @@ -0,0 +1,34 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +examples_env = env.Clone(); +examples_env['CXX'] = 'syclcc-clang'; +examples_env['CXXFLAGS'] += ['-O3']; + +examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += examples_env.sources; +env.headers += examples_env.headers; + +# Cavity2D +examples_objects = []; +examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); +examples_env.poiseulle_2d_gpu = examples_env.Program('#bin/vawt_3d_hlbm_gpu', [examples_objects]); + +# Set Alias +env.examples = [ + examples_env.poiseulle_2d_gpu +]; +env.Alias('examples', env.examples); +env.targets += ['examples']; +env.Install('$prefix/bin/', env.examples); diff --git a/examples/vawt_3d_hlbm_gpu/SConstruct b/examples/vawt_3d_hlbm_gpu/SConstruct new file mode 100644 index 0000000..0611b67 --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/SConstruct @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env_vars.Add('build_examples', + help='If examples should be built', + default="true" +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=[ + '-std=c++20', + '-g', + '-Wall', + '-Wextra' + ], + LIBS=[ + 'forstio-core', + 'forstio-async', + 'forstio-io' + ] +); +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/examples/vawt_3d_hlbm_gpu/sim.cpp b/examples/vawt_3d_hlbm_gpu/sim.cpp new file mode 100644 index 0000000..28d404b --- /dev/null +++ b/examples/vawt_3d_hlbm_gpu/sim.cpp @@ -0,0 +1,656 @@ +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace kel { +namespace lbm { + +// Deliberately tiny: 32 x 32 x 32 = 32768 cells, so a full run finishes in +// seconds. Square in y/z so the channel cross section is symmetric about the +// particle, which makes a stray transverse force obvious. +constexpr uint64_t dim_x = 32ul; +constexpr uint64_t dim_y = 32ul; +constexpr uint64_t dim_z = 32ul; + +constexpr uint64_t particle_amount = 1ul; + +/** + * The particle lives in lattice space, exactly like the fluid: the radius is a + * number of cells, the density is relative to the fluid's rho = 1, and the + * mass, inertia, force and torque that follow are all lattice quantities. The + * converter handles the fluid's SI <-> LBM bookkeeping and is deliberately not + * applied to any of this -- so every column of particle.csv is in cells, + * steps, and lattice mass. + */ +constexpr double particle_radius = 0.15 * static_cast(dim_y); +constexpr double particle_density = 1.0; + +namespace sch { + +/** + * The spin axis is an arbitrary fixed vector, not a lattice direction. It is + * normalised by the integrator, so these three need not be a unit vector. + * + * Defaults to +y: flow along x, axis vertical and perpendicular to it, blades + * sweeping the x-z plane -- the vertical-axis turbine arrangement. + */ +using LbmArgs = Args< + Struct< + Member, + Member, + Member, + Member, + Member, + Member + >, + Tuple<> +>; + +using InfoChunk = Chunk; + +template +using DfChunk = Chunk, 1u, dim_x, dim_y, dim_z>; + +template +using ScalarChunk = Chunk, 0u, dim_x, dim_y, dim_z>; + +template +using VectorChunk = Chunk, 0u, dim_x, dim_y, dim_z>; + +template +using ChunkStruct = Struct< + Member, + Member, "dfs">, + Member, "dfs_old">, + Member, "particle_N">, + Member, "particle_D"> +>; + +template +using MomChunk = Chunk, 0u, dim_x, dim_y, dim_z>; + +template +using RhoChunk = Chunk, 0u, dim_x, dim_y, dim_z>; + +/** + * "force" is not optional here: HlbmOneParticleGather reads it, so the 3D + * macro struct needs it the moment a particle is carried. + */ +template +using MacroStruct = Struct< + Member, "velocity">, + Member, "density">, + Member, "force">, + Member, "porosity"> +>; + +/** + * particle_aabb is specialised only on ParticleGroup, so a bare + * FixedArray> hits its always_false primary template. + */ +template +using ParticleSpheroidGroup = ParticleGroup>; +} + +template +kel::lbm::error_or setup_initial_conditions( + converter& conv, + kel::lbm::data>& fields, + kel::lbm::data>& macros, + kel::lbm::data>& particles +){ + // Unit conversion is not exercised in this reduced case. + (void) conv; + + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u,1u}} + ); + + // Inflow + iterator::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u,0u}}, + {{1u,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + + // Outflow + iterator::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u,0u}}, + {{dim_x,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& mom_f = macros.template get<"velocity">(); + auto& por_f = macros.template get<"porosity">(); + auto& force_f = macros.template get<"force">(); + + iterator::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + por_f.at(index).at({}) = {1}; + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + auto eq = equilibrium(rho,mom); + + df = eq; + + auto& force = force_f.at(index); + for(uint64_t i{0u}; i < Desc::D; ++i){ + force.at({{i}}).set(0); + } + }, + {},// 0-index + df_f.get_dims() + ); + + iterator::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + mom.at({{0u}}) = 0.01; + auto eq = equilibrium(rho,mom); + + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u,1u}} + ); + + kel::lbm::data>> rad; + rad.at({}).set(particle_radius); + kel::lbm::data> dense; + dense.at({}).set(particle_density); + + // Mask resolution 16 in 3D is 16^3 = 4096 voxels. The analytic spheroid + // porosity is what HLBM actually calls, but the mask is still filled. + particles = create_spheroid_particle_group(rad,dense,{16u}); + { + auto& rb = particles.template get<"particles">().at({0u}).template get<"rigid_body">(); + auto& rbp = rb.template get<"position">(); + rbp.at({{0u}}) = 0.3 * dim_x; + rbp.at({{1u}}) = 0.5 * dim_y; + rbp.at({{2u}}) = 0.5 * dim_z; + + // position_old == position means the body starts at rest. It also + // makes a pinned body exact later: 2x - x_old + 0 == x. + rb.template get<"position_old">() = rbp; + + kel::lbm::data> zero; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero.at({{i}}).set(0); + } + // Acceleration is unit typed now, so its zero has to be too. + kel::lbm::data>> zero_acc; + for(uint64_t i{0u}; i < Desc::D; ++i){ + zero_acc.at({{i}}).set(0); + } + rb.template get<"acceleration">() = zero_acc; + + // In 3D these are Vector (a rotation vector), not a scalar angle. + rb.template get<"rotation">() = zero; + rb.template get<"rotation_old">() = zero; + rb.template get<"angular_acceleration">() = zero; + } + + return kel::lbm::make_void(); +} + +template +kel::lbm::error_or step( + converter& conv, + kel::lbm::data>,encode::Sycl>& fields, + kel::lbm::data>,encode::Sycl>& macros, + kel::lbm::data>,encode::Sycl>& particles, + kel::lbm::data t_i, + kel::lbm::data sub_steps, + const kel::lbm::data>& spin_axis, + device& dev +){ + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); + + // Porosity back to 1 (all fluid) before the particle restamps itself. + q.submit([&](acpp::sycl::handler& h){ + component> hlbm_reset; + + h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + hlbm_reset.apply(fields,macros,index,t_i); + }); + }).wait(); + + /** + * One work item per particle. The stamp has to come after the integrate: + * it writes the post-integration surface velocity back onto the lattice. + */ + q.submit([&](acpp::sycl::handler& h){ + component> p_reset; + component> p_gather; + component> p_stamp; + + /** + * The rotor replaces the free-body integrator. No contact pass: the + * body is pinned, so it cannot reach a wall or another particle. + */ + component> p_integrate{spin_axis}; + + h.parallel_for(acpp::sycl::range<1u>{particle_amount}, [=](acpp::sycl::id<1u> idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < 1u; ++i){ + index.at({{i}}).set(idx[i]); + } + + p_reset.apply(particles,index,t_i); + p_gather.apply(fields,macros,particles,index,t_i,sub_steps); + p_integrate.apply(particles,index,t_i,sub_steps); + p_stamp.apply(fields,macros,particles,index,t_i,sub_steps); + }); + }).wait(); + + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + /** + * Hlbm rather than BGK: it is the collision that consumes the + * porosity and particle_N/particle_D the stamp pass writes. + */ + component> collision{conv.template kinematic_viscosity_si_to_tau({0.1})}; + component> bb; + + component,encode::Sycl> flow_in{ + [&](){ + uint64_t target_t_i = 256u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.001 / target_t_i) * t_i.get(); + } + return 1.001; + }() + }; + component,encode::Sycl> flow_out{1.0}; + + h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: + break; + case 1u: + bb.apply(fields,index,t_i); + break; + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 3u: + flow_in.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: + break; + } + }); + }).wait(); + + q.submit([&](acpp::sycl::handler& h){ + component> stream; + + h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ + kel::lbm::data> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(fields,index,t_i); + } + }); + }).wait(); + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ + + return kel::lbm::make_void(); +} +} +} + +template +kel::lbm::error_or lbm_main(int argc, char** argv){ + using namespace kel::lbm; + + kel::lbm::data args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("vawt_3d_hlbm_gpu"); + auto& steps_arg = args_n.template get<"time_steps">(); + steps_arg.set(256u); + auto& sub_arg = args_n.template get<"sub_steps">(); + sub_arg.set(16u); + + // +y by default: flow along x, axis perpendicular to it. + args_n.template get<"axis_x">().set(0.0); + args_n.template get<"axis_y">().set(1.0); + args_n.template get<"axis_z">().set(0.0); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } + } + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); + } + const auto& lbm_env = eo_lbm_env.get_value(); + + auto& out_dir = lbm_env.name_dir; + + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error("Could not create output directory"); + } + } + + converter conv { + // delta_x + {1.0}, + // delta_t + {1.0} + }; + + // Characteristic length is the body diameter -- the scale that sets Re here. + print_lbm_meta(conv,{0.1},{0.01},{2.0 * particle_radius}); + + auto lbm_data_ptr = kel::lbm::heap>>(); + auto lbm_macro_data_ptr = kel::lbm::heap>>(); + auto lbm_particle_data_ptr = kel::lbm::heap>>(); + + std::cout<<"Estimated Bytes: "<,sch::MacroStruct>().get()<( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; + + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); + + device dev; + + auto& sycl_q = dev.get_handle(); + + sycl_q.wait(); + { + auto eov = setup_initial_conditions(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + kel::lbm::data, encode::Sycl> lbm_sycl_data{sycl_q}; + kel::lbm::data, encode::Sycl> lbm_sycl_macro_data{sycl_q}; + kel::lbm::data, encode::Sycl> lbm_sycl_particle_data{sycl_q}; + sycl_q.wait(); + + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + auto lsdp_view = make_view(lbm_sycl_particle_data); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + kel::lbm::data time_steps = args.template get<"args">().template get<"time_steps">(); + kel::lbm::data sub_steps = args.template get<"args">().template get<"sub_steps">(); + + kel::lbm::data> spin_axis; + { + auto& args_n = args.template get<"args">(); + spin_axis.at({{0u}}).set(args_n.template get<"axis_x">().get()); + spin_axis.at({{1u}}).set(args_n.template get<"axis_y">().get()); + spin_axis.at({{2u}}).set(args_n.template get<"axis_z">().get()); + } + + // Unit axis, for projecting the rotation and torque onto it below. + auto axis_hat = kel::lbm::math::normalize(spin_axis); + + /** + * Particle time series. write_vtk_file only accepts a struct of chunks, so + * a particle group cannot go through it -- a CSV beside the VTK output is + * the simpler answer, and it is what you actually plot. + * + * Velocity and angular velocity are not stored on the body: the scheme is + * position-Verlet, so both are recovered as (now - previous) * steps. + */ + std::ofstream particle_csv{out_dir / "particle.csv"}; + if(not particle_csv.is_open()){ + return kel::lbm::make_error("Could not open particle.csv"); + } + /** + * theta/omega/alpha/tau are the rotation, angular velocity, angular + * acceleration and torque projected onto the spin axis -- the rotor + * numbers. drag is the streamwise force. The raw vectors follow so a + * stray off-axis component is still visible rather than hidden by the + * projection. + */ + particle_csv + <<"step,theta,omega,alpha,tau,drag," + <<"x,y,z,vx,vy,vz,fx,fy,fz," + <<"rot_x,rot_y,rot_z,omega_x,omega_y,omega_z," + <<"alpha_x,alpha_y,alpha_z,tau_x,tau_y,tau_z\n"; + + auto write_particle_row = [&](uint64_t step_i){ + auto& rb = lbm_particle_data_ptr->template get<"particles">() + .at({0u}).template get<"rigid_body">(); + + auto& pos = rb.template get<"position">(); + auto& pos_old = rb.template get<"position_old">(); + auto& force = rb.template get<"force">(); + auto& rot = rb.template get<"rotation">(); + auto& rot_old = rb.template get<"rotation_old">(); + auto& ang_acc = rb.template get<"angular_acceleration">(); + auto& torque = rb.template get<"torque">(); + + const auto steps = static_cast(particle_sub_steps(sub_steps)); + + // Project onto the spin axis. + double theta = 0.0, omega = 0.0, alpha = 0.0, tau = 0.0; + for(uint64_t k{0u}; k < 3u; ++k){ + const double a = axis_hat.at({{k}}).get(); + theta += rot.at({{k}}).get() * a; + omega += (rot.at({{k}}).get() - rot_old.at({{k}}).get()) * steps * a; + alpha += ang_acc.at({{k}}).get() * a; + tau += torque.at({{k}}).get() * a; + } + + particle_csv + < i{0u}; i < time_steps and krun; ++i){ + { + { + auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + { + auto eov = step(conv,lsd_view,lsdm_view,lsdp_view,i,sub_steps,spin_axis,dev); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_particle_data,*lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + } + write_particle_row(i.get()); + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + wait.poll(); + if(print_status){ + std::cout<<"Status: "<().get() * 100 / time_steps.get())<<"%"<::apply( + [&](auto& index){ + std::cout<().at(index).template cast_to().get()<<"\n"; + }, + {{0u,0u}}, + {{dim_x, dim_y}} + ); + */ + + sycl_q.wait(); + return kel::lbm::make_void(); +} + +using FloatT = kel::lbm::sch::Float32; + +int main(int argc, char** argv){ + auto eov = lbm_main(argc, argv); + if(eov.is_error()){ + auto& err = eov.get_error(); + std::cerr<<"[Error] "< 0u){ + std::cerr<<" - "<