#include #include #include #include #include #include #include #include namespace kel { namespace lbm { // Deliberately tiny: 32 x 32 x 32 = 32768 cells, so a full run finishes in // seconds. Square in y/z so the channel cross section is symmetric about the // particle, which makes a stray transverse force obvious. constexpr uint64_t dim_x = 32ul; constexpr uint64_t dim_y = 32ul; constexpr uint64_t dim_z = 32ul; constexpr uint64_t particle_amount = 1ul; /** * The particle lives in lattice space, exactly like the fluid: the radius is a * number of cells, the density is relative to the fluid's rho = 1, and the * mass, inertia, force and torque that follow are all lattice quantities. The * converter handles the fluid's SI <-> LBM bookkeeping and is deliberately not * applied to any of this -- so every column of particle.csv is in cells, * steps, and lattice mass. */ constexpr double particle_radius = 0.15 * static_cast(dim_y); constexpr double particle_density = 1.0; namespace sch { using LbmArgs = Args< Struct< Member, Member, Member >, Tuple<> >; using InfoChunk = Chunk; template using DfChunk = Chunk, 1u, dim_x, dim_y, dim_z>; template using ScalarChunk = Chunk, 0u, dim_x, dim_y, dim_z>; template using VectorChunk = Chunk, 0u, dim_x, dim_y, dim_z>; template using ChunkStruct = Struct< Member, Member, "dfs">, Member, "dfs_old">, Member, "particle_N">, Member, "particle_D"> >; template using MomChunk = Chunk, 0u, dim_x, dim_y, dim_z>; template using RhoChunk = Chunk, 0u, dim_x, dim_y, dim_z>; /** * "force" is not optional here: HlbmOneParticleGather reads it, so the 3D * macro struct needs it the moment a particle is carried. */ template using MacroStruct = Struct< Member, "velocity">, Member, "density">, Member, "force">, Member, "porosity"> >; /** * particle_aabb is specialised only on ParticleGroup, so a bare * FixedArray> hits its always_false primary template. */ template using ParticleSpheroidGroup = ParticleGroup>; } template kel::lbm::error_or setup_initial_conditions( converter& conv, kel::lbm::data>& fields, kel::lbm::data>& macros, kel::lbm::data>& particles ){ // Unit conversion is not exercised in this reduced case. (void) conv; auto& info_f = fields.template get<"info">(); // Set everything as walls iterator::apply( [&](auto& index){ info_f.at(index).set(1u); }, {}, info_f.get_dims(), {} ); // Fluid iterator::apply( [&](auto& index){ info_f.at(index).set(2u); }, {}, info_f.get_dims(), {{1u,1u,1u}} ); // Inflow iterator::apply( [&](auto& index){ info_f.at(index).set(3u); }, {{0u,0u,0u}}, {{1u,dim_y,dim_z}}, {{0u,1u,1u}} ); // Outflow iterator::apply( [&](auto& index){ info_f.at(index).set(4u); }, {{dim_x-1u,0u,0u}}, {{dim_x,dim_y,dim_z}}, {{0u,1u,1u}} ); // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); auto& mom_f = macros.template get<"velocity">(); auto& por_f = macros.template get<"porosity">(); auto& force_f = macros.template get<"force">(); iterator::apply( [&](auto& index){ auto& df = df_f.at(index); auto& rho = rho_f.at(index); por_f.at(index).at({}) = {1}; rho.at({}) = {1}; auto& mom = mom_f.at(index); auto eq = equilibrium(rho,mom); df = eq; auto& force = force_f.at(index); for(uint64_t i{0u}; i < Desc::D; ++i){ force.at({{i}}).set(0); } }, {},// 0-index df_f.get_dims() ); iterator::apply( [&](auto& index){ auto& df = df_f.at(index); auto& rho = rho_f.at(index); rho.at({}) = {1}; auto& mom = mom_f.at(index); mom.at({{0u}}) = 0.01; auto eq = equilibrium(rho,mom); df = eq; }, {},// 0-index df_f.get_dims(), {{1u,1u,1u}} ); kel::lbm::data>> rad; rad.at({}).set(particle_radius); kel::lbm::data> dense; dense.at({}).set(particle_density); // Mask resolution 16 in 3D is 16^3 = 4096 voxels. The analytic spheroid // porosity is what HLBM actually calls, but the mask is still filled. particles = create_spheroid_particle_group(rad,dense,{16u}); { auto& rb = particles.template get<"particles">().at({0u}).template get<"rigid_body">(); auto& rbp = rb.template get<"position">(); rbp.at({{0u}}) = 0.3 * dim_x; rbp.at({{1u}}) = 0.5 * dim_y; rbp.at({{2u}}) = 0.5 * dim_z; // position_old == position means the body starts at rest. It also // makes a pinned body exact later: 2x - x_old + 0 == x. rb.template get<"position_old">() = rbp; kel::lbm::data> zero; for(uint64_t i{0u}; i < Desc::D; ++i){ zero.at({{i}}).set(0); } // Acceleration is unit typed now, so its zero has to be too. kel::lbm::data>> zero_acc; for(uint64_t i{0u}; i < Desc::D; ++i){ zero_acc.at({{i}}).set(0); } rb.template get<"acceleration">() = zero_acc; // In 3D these are Vector (a rotation vector), not a scalar angle. rb.template get<"rotation">() = zero; rb.template get<"rotation_old">() = zero; rb.template get<"angular_acceleration">() = zero; } return kel::lbm::make_void(); } template kel::lbm::error_or step( converter& conv, kel::lbm::data>,encode::Sycl>& fields, kel::lbm::data>,encode::Sycl>& macros, kel::lbm::data>,encode::Sycl>& particles, kel::lbm::data t_i, kel::lbm::data sub_steps, device& dev ){ auto& q = dev.get_handle(); auto& info_f = fields.template get<"info">(); // Porosity back to 1 (all fluid) before the particle restamps itself. q.submit([&](acpp::sycl::handler& h){ component> hlbm_reset; h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < Desc::D; ++i){ index.at({{i}}).set(idx[i]); } hlbm_reset.apply(fields,macros,index,t_i); }); }).wait(); /** * One work item per particle. The stamp has to come after the integrate: * it writes the post-integration surface velocity back onto the lattice. */ q.submit([&](acpp::sycl::handler& h){ component> p_reset; component> p_gather; component> p_integrate; component> p_stamp; // Soft shell against the domain walls so the sphere cannot tunnel out. dem_params p_dem{1.0e-2, 0.3, 0.0, 0.0}; kel::lbm::data> p_extent{{dim_x,dim_y,dim_z}}; component> p_contact{p_dem,p_extent}; h.parallel_for(acpp::sycl::range<1u>{particle_amount}, [=](acpp::sycl::id<1u> idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < 1u; ++i){ index.at({{i}}).set(idx[i]); } p_reset.apply(particles,index,t_i); p_gather.apply(fields,macros,particles,index,t_i,sub_steps); p_contact.apply(particles,index,t_i,sub_steps); p_integrate.apply(particles,index,t_i,sub_steps); p_stamp.apply(fields,macros,particles,index,t_i,sub_steps); }); }).wait(); // auto coll_ev = q.submit([&](acpp::sycl::handler& h){ /** * Hlbm rather than BGK: it is the collision that consumes the * porosity and particle_N/particle_D the stamp pass writes. */ component> collision{conv.template kinematic_viscosity_si_to_tau({0.1})}; component> bb; component,encode::Sycl> flow_in{ [&](){ uint64_t target_t_i = 256u; if(t_i.get() < target_t_i){ return 1.0 + (0.001 / target_t_i) * t_i.get(); } return 1.001; }() }; component,encode::Sycl> flow_out{1.0}; h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < Desc::D; ++i){ index.at({{i}}).set(idx[i]); } auto info = info_f.at(index); switch(info.get()){ case 0u: break; case 1u: bb.apply(fields,index,t_i); break; case 2u: collision.apply(fields,macros,index,t_i); break; case 3u: flow_in.apply(fields,index,t_i); collision.apply(fields,macros,index,t_i); break; case 4u: flow_out.apply(fields,index,t_i); collision.apply(fields,macros,index,t_i); break; default: break; } }); }).wait(); q.submit([&](acpp::sycl::handler& h){ component> stream; h.parallel_for(acpp::sycl::range{dim_x,dim_y,dim_z}, [=](acpp::sycl::id idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < Desc::D; ++i){ index.at({{i}}).set(idx[i]); } auto info = info_f.at(index); if(info.get() > 0u){ stream.apply(fields,index,t_i); } }); }).wait(); // Step /* q.submit([&](acpp::sycl::handler& h){ // h.depends_on(collision_ev); }).wait(); */ return kel::lbm::make_void(); } } } template kel::lbm::error_or lbm_main(int argc, char** argv){ using namespace kel::lbm; kel::lbm::data args; { auto& args_n = args.template get<"args">(); auto& name = args_n.template get<"name">(); name.set("poiseulle_particle_3d_hlbm_gpu"); auto& steps_arg = args_n.template get<"time_steps">(); steps_arg.set(256u); auto& sub_arg = args_n.template get<"sub_steps">(); sub_arg.set(16u); } { auto eov = kel::lbm::parse_args(args,argc,argv); if(eov.is_error()){ return eov; } } auto eo_lbm_env = init_lbm_env(args); if(eo_lbm_env.is_error()){ return std::move(eo_lbm_env.get_error()); } const auto& lbm_env = eo_lbm_env.get_value(); auto& out_dir = lbm_env.name_dir; { std::error_code ec; std::filesystem::create_directories(out_dir,ec); if(ec != std::errc{}){ return kel::lbm::make_error("Could not create output directory"); } } converter conv { // delta_x {1.0}, // delta_t {1.0} }; // Characteristic length is the body diameter -- the scale that sets Re here. print_lbm_meta(conv,{0.1},{0.01},{2.0 * particle_radius}); auto lbm_data_ptr = kel::lbm::heap>>(); auto lbm_macro_data_ptr = kel::lbm::heap>>(); auto lbm_particle_data_ptr = kel::lbm::heap>>(); std::cout<<"Estimated Bytes: "<,sch::MacroStruct>().get()<( std::string{eo_aio.get_error().get_message()}); } auto& aio = eo_aio.get_value(); saw::wait_scope wait{aio.event_loop}; bool krun = true; bool print_status = false; aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ krun = false; }).detach(); aio.event_port.on_signal(saw::Signal::User1).then([&](){ print_status = true; }).detach(); device dev; auto& sycl_q = dev.get_handle(); sycl_q.wait(); { auto eov = setup_initial_conditions(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); if(eov.is_error()){ return eov; } } kel::lbm::data, encode::Sycl> lbm_sycl_data{sycl_q}; kel::lbm::data, encode::Sycl> lbm_sycl_macro_data{sycl_q}; kel::lbm::data, encode::Sycl> lbm_sycl_particle_data{sycl_q}; sycl_q.wait(); auto lsd_view = make_view(lbm_sycl_data); auto lsdm_view = make_view(lbm_sycl_macro_data); auto lsdp_view = make_view(lbm_sycl_particle_data); { auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); if(eov.is_error()){ return eov; } } { auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); if(eov.is_error()){ return eov; } } { auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); if(eov.is_error()){ return eov; } } sycl_q.wait(); kel::lbm::data time_steps = args.template get<"args">().template get<"time_steps">(); kel::lbm::data sub_steps = args.template get<"args">().template get<"sub_steps">(); /** * Particle time series. write_vtk_file only accepts a struct of chunks, so * a particle group cannot go through it -- a CSV beside the VTK output is * the simpler answer, and it is what you actually plot. * * Velocity and angular velocity are not stored on the body: the scheme is * position-Verlet, so both are recovered as (now - previous) * steps. */ std::ofstream particle_csv{out_dir / "particle.csv"}; if(not particle_csv.is_open()){ return kel::lbm::make_error("Could not open particle.csv"); } particle_csv <<"step,x,y,z,vx,vy,vz,fx,fy,fz," <<"rot_x,rot_y,rot_z,omega_x,omega_y,omega_z," <<"alpha_x,alpha_y,alpha_z,tau_x,tau_y,tau_z\n"; auto write_particle_row = [&](uint64_t step_i){ auto& rb = lbm_particle_data_ptr->template get<"particles">() .at({0u}).template get<"rigid_body">(); auto& pos = rb.template get<"position">(); auto& pos_old = rb.template get<"position_old">(); auto& force = rb.template get<"force">(); auto& rot = rb.template get<"rotation">(); auto& rot_old = rb.template get<"rotation_old">(); auto& ang_acc = rb.template get<"angular_acceleration">(); auto& torque = rb.template get<"torque">(); const auto steps = static_cast(particle_sub_steps(sub_steps)); particle_csv< i{0u}; i < time_steps and krun; ++i){ { { auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); if(eov.is_error()){ return eov; } } } { auto eov = step(conv,lsd_view,lsdm_view,lsdp_view,i,sub_steps,dev); if(eov.is_error()){ return eov; } } { auto eov = dev.copy_to_host(lbm_sycl_particle_data,*lbm_particle_data_ptr); if(eov.is_error()){ return eov; } } write_particle_row(i.get()); { auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); if(eov.is_error()){ return eov; } } { auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); if(eov.is_error()){ return eov; } } wait.poll(); if(print_status){ std::cout<<"Status: "<().get() * 100 / time_steps.get())<<"%"<::apply( [&](auto& index){ std::cout<().at(index).template cast_to().get()<<"\n"; }, {{0u,0u}}, {{dim_x, dim_y}} ); */ sycl_q.wait(); return kel::lbm::make_void(); } using FloatT = kel::lbm::sch::Float32; int main(int argc, char** argv){ auto eov = lbm_main(argc, argv); if(eov.is_error()){ auto& err = eov.get_error(); std::cerr<<"[Error] "< 0u){ std::cerr<<" - "<