#include #include #include #include #include #include #include #include namespace kel { namespace lbm { constexpr uint64_t dim_x = 512ul; constexpr uint64_t dim_y = dim_x * 2ul; constexpr uint64_t particle_amount = 16ul; namespace sch { using InfoChunk = Chunk; template using DfChunk = Chunk, 1u, dim_x, dim_y>; template using ScalarChunk = Chunk, 0u, dim_x, dim_y>; template using VectorChunk = Chunk, 0u, dim_x, dim_y>; template using ChunkStruct = Struct< Member, Member, "dfs">, Member, "dfs_old">, Member, "particle_N">, Member, "particle_D"> >; template using VelChunk = Chunk, 0u, dim_x, dim_y>; template using RhoChunk = Chunk, 0u, dim_x, dim_y>; template using PorChunk = Chunk, 0u, dim_x, dim_y>; template using MacroStruct = Struct< Member, "velocity">, Member, "density">, // HlbmOneParticleGather reads this one. Member, "force">, Member, "porosity"> >; template using ParticleSpheroidGroup = ParticleGroup< T, Desc::D, particle_amount, coll::Spheroid >; } template kel::lbm::error_or setup_initial_conditions( kel::lbm::data>& fields, kel::lbm::data>& macros, kel::lbm::data>& p_group ){ auto& info_f = fields.template get<"info">(); // Set everything as walls iterator::apply( [&](auto& index){ info_f.at(index).set(1u); }, {}, info_f.get_dims(), {} ); // Fluid iterator::apply( [&](auto& index){ info_f.at(index).set(2u); }, {}, info_f.get_dims(), {{1u,1u}} ); // auto& df_f = fields.template get<"dfs_old">(); auto& rho_f = macros.template get<"density">(); auto& vel_f = macros.template get<"velocity">(); iterator::apply( [&](auto& index){ auto& df = df_f.at(index); auto& rho = rho_f.at(index); rho.at({}) = {1}; auto& vel = vel_f.at(index); auto eq = equilibrium(rho,vel); df = eq; }, {},// 0-index df_f.get_dims() ); // Particles { // Radius is per particle now, and a lattice length. kel::lbm::data>> rad_p; rad_p.at({}).set(2.0); kel::lbm::data> dense_p; dense_p.at({}).set(1); // auto& spheroid_group = particles.template get<0u>(); auto& spheroid_group = p_group; spheroid_group = create_spheroid_particle_group( rad_p, dense_p, {64u} ); { auto& p = spheroid_group.template get<"particles">(); iterator<1u>::apply( [&](auto& index){ // Set Pos here? auto& p_ind = p.at(index); auto& p_rb = p_ind.template get<"rigid_body">(); auto& p_pos = p_rb.template get<"position">(); kel::lbm::data> p; uint64_t x = index.at({0u}).get() % 8u; uint64_t y = index.at({1u}).get() / 8u; p.at({{0u}}).set( 0.2 + 0.6 / (x-1u) ); p.at({{1u}}).set( 0.8 + y * 0.1 ); p_pos.handle() = p; auto& p_pos_old = p_rb.template get<"position_old">(); p_pos_old = p_pos; }, {}, p.get_dims() ); } } // Particle in hacky flavour {} return kel::lbm::make_void(); } template kel::lbm::error_or step( kel::lbm::data>,encode::Sycl>& fields, kel::lbm::data>,encode::Sycl>& macros, kel::lbm::data>,encode::Sycl>& p_group, kel::lbm::data t_i, device& dev ){ auto& q = dev.get_handle(); auto& info_f = fields.template get<"info">(); auto& parts = p_group.template get<"particles">(); auto& p_mask = p_group.template get<"mask">(); auto& vels = macros.template get<"velocity">(); auto& forces = macros.template get<"force">(); auto p_meta = parts.meta(); q.submit([&](acpp::sycl::handler& h){ h.parallel_for(acpp::sycl::range{dim_x,dim_y}, [=](acpp::sycl::id idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < Desc::D; ++i){ index.at({{i}}).set(idx[i]); } }); }).wait(); q.submit([&](acpp::sycl::handler& h){ h.parallel_for(acpp::sycl::range<1u>{p_meta.at({0u}).get()}, [=](acpp::sycl::id<1u> idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < 1u; ++i){ index.at({{i}}).set(idx[i]); } auto& pi = parts.at(index); auto& pirb = pi.template get<"rigid_body">(); auto& p_pos = pirb.template get<"position">(); auto& p_rot = pirb.template get<"rotation">(); auto& p_acc = pirb.template get<"acceleration">(); kel::lbm::data>> delta_t; delta_t.at({}).set(1.0f); verlet_step_lambda(pi,delta_t); }); }).wait(); // auto coll_ev = q.submit([&](acpp::sycl::handler& h){ // Need nicer things to handle the flow. I see improvement here component> collision{0.8}; component> bb; h.parallel_for(acpp::sycl::range{dim_x,dim_y}, [=](acpp::sycl::id idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < Desc::D; ++i){ index.at({{i}}).set(idx[i]); } auto info = info_f.at(index); switch(info.get()){ case 0u: break; case 1u: bb.apply(fields,index,t_i); break; case 2u: collision.apply(fields,macros,index,t_i); break; default: break; } }); }).wait(); // Step /* q.submit([&](acpp::sycl::handler& h){ // h.depends_on(collision_ev); }).wait(); */ return kel::lbm::make_void(); } } } template kel::lbm::error_or lbm_main(int argc, char** argv){ using namespace kel::lbm; using dfi = df_info; auto eo_lbm_dir = output_directory(); if(eo_lbm_dir.is_error()){ return std::move(eo_lbm_dir.get_error()); } auto& lbm_dir = eo_lbm_dir.get_value(); auto out_dir = lbm_dir / "settling_spheres_2d_hlbm_gpu"; { std::error_code ec; std::filesystem::create_directories(out_dir,ec); if(ec != std::errc{}){ return kel::lbm::make_error("Could not create output directory"); } } converter conv { // delta_x {1.0}, // delta_t {1.0} }; print_lbm_meta(conv,{0.05},{0.01},{0.4 * dim_y}); // kel::lbm::data> meta{{dim_x,dim_y}}; auto lbm_data_ptr = kel::lbm::heap>>(); auto lbm_macro_data_ptr = kel::lbm::heap>>(); auto lbm_particle_data_ptr = kel::lbm::heap>>(); // auto lbm_particles_ptr = kel::lbm::heap,part_count>>>(); // kel::lbm::data> p_mask; std::cout<<"Estimated Bytes of LBM Fields: "<,sch::MacroStruct>().get()<( std::string{eo_aio.get_error().get_message()}); } auto& aio = eo_aio.get_value(); saw::wait_scope wait{aio.event_loop}; bool krun = true; bool print_status = false; aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ krun = false; }).detach(); aio.event_port.on_signal(saw::Signal::User1).then([&](){ print_status = true; }).detach(); device dev; auto& sycl_q = dev.get_handle(); sycl_q.wait(); { auto eov = setup_initial_conditions(*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); if(eov.is_error()){ return eov; } } { auto eov = write_vtk_file(out_dir,"initial_state",0u,*lbm_data_ptr); if(eov.is_error()){ return eov; } } kel::lbm::data, encode::Sycl> lbm_sycl_data{sycl_q}; kel::lbm::data, encode::Sycl> lbm_sycl_macro_data{sycl_q}; kel::lbm::data, encode::Sycl> lbm_sycl_particle_data{sycl_q}; { auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); if(eov.is_error()){ return eov; } } { auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); if(eov.is_error()){ return eov; } } { auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); if(eov.is_error()){ return eov; } } sycl_q.wait(); auto lsd_view = make_view(lbm_sycl_data); auto lsdm_view = make_view(lbm_sycl_macro_data); auto lsdp_view = make_view(lbm_sycl_particle_data); { auto eov = write_vtk_file(out_dir,"ms",0u, *lbm_macro_data_ptr); if(eov.is_error()){ return eov; } } kel::lbm::data time_steps{16u*4096ul}; auto& info_f = lsd_view.template get<"info">(); for(kel::lbm::data i{0u}; i < time_steps and krun; ++i){ // BC + Collision { auto eov = step(lsd_view,lsdm_view,lsdp_view,i,dev); if(eov.is_error()){ return eov; } } sycl_q.wait(); if(i.get()%1u ==0u){ { auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); if(eov.is_error()){ return eov; } } { auto eov = write_vtk_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); if(eov.is_error()){ return eov; } } } /* { { auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); if(eov.is_error()){ return eov; } } { auto eov = write_csv_file(out_dir,"lbm",i.get(), *lbm_data_ptr); if(eov.is_error()){ return eov; } } } */ // Stream sycl_q.submit([&](acpp::sycl::handler& h){ component> stream; h.parallel_for(acpp::sycl::range{dim_x,dim_y}, [=](acpp::sycl::id idx){ kel::lbm::data> index; for(uint64_t i = 0u; i < Desc::D; ++i){ index.at({{i}}).set(idx[i]); } auto info = info_f.at(index); if(info.get() > 0u){ stream.apply(lsd_view,index,i); } }); }).wait(); wait.poll(); if(print_status){ std::cout<<"Status: "<().get() * 100 / time_steps.get())<<"%"<(argc, argv); if(eov.is_error()){ auto& err = eov.get_error(); std::cerr<<"[Error] "< 0u){ std::cerr<<" - "<