diff options
| author | Claudius "keldu" Holeksa <mail@keldu.de> | 2026-11-13 14:12:49 +0100 |
|---|---|---|
| committer | Claudius "keldu" Holeksa <mail@keldu.de> | 2026-11-13 14:12:49 +0100 |
| commit | e821d617802ebbfff756844e4d9720342fc78a4e (patch) | |
| tree | 1dcf33b84698b5074218e44ea491b77a9c23ee66 | |
| parent | 0c0087ef9482634273ec1a23d3a915754f94926d (diff) | |
| download | libs-lbm-dev.tar.gz | |
Swarm POCdev
94 files changed, 9138 insertions, 2589 deletions
diff --git a/.nix/adaptive-cpp.nix b/.nix/adaptive-cpp.nix index 2e631e1..34d079a 100644 --- a/.nix/adaptive-cpp.nix +++ b/.nix/adaptive-cpp.nix @@ -11,8 +11,10 @@ , python3 , config ? { rocm = false; + cuda = false; } , rocmPackages +, cudaPackages , spirv-tools , spirv-headers , spirv-llvm-translator @@ -50,21 +52,31 @@ in stdenv.mkDerivation ({ spirv-tools spirv-headers spirv-llvm-translator + ] ++ lib.optionals config.cuda [ + cudaPackages.cuda_cudart + cudaPackages.cuda_nvcc ]; cmakeFlags = [ "-DCLANG_INCLUDE_PATH=${llvmPackages.libclang.dev}/include" ] ++ lib.optionals config.rocm [ "-DWITH_ROCM_BACKEND=ON" + ] ++ lib.optionals config.cuda [ + "-DWITH_CUDA_BACKEND=ON" + # nixpkgs splits the toolkit across outputs, so point cmake at the joined + # root rather than letting it search /usr/local/cuda. + "-DCUDA_TOOLKIT_ROOT_DIR=${cudaPackages.cudatoolkit}" + ] ++ lib.optionals (config.rocm || config.cuda) [ # Keep OpenCL and Level Zero off. src/CMakeLists.txt turns WITH_LLVM_TO_SPIRV # on for either of them, and that target FetchContent-clones # SPIRV-LLVM-Translator at configure time, which a nix build cannot do -- - # it has neither git nor network. ROCm is the backend we want here and it - # does not go through SPIR-V. + # it has neither git nor network. Neither ROCm (amdgcn) nor CUDA (PTX) goes + # through SPIR-V, so turning them off costs nothing here. # - # This only bites when rocm = true: rocmPackages.clr brings OpenCL in, so - # the backend autodetects on, where in the base build it stays off. + # This only bites with a GPU backend on: rocmPackages.clr and the CUDA + # toolkit both bring OpenCL in, so the backend autodetects on, where in the + # base build it stays off. "-DWITH_OPENCL_BACKEND=OFF" "-DWITH_LEVEL_ZERO_BACKEND=OFF" ]; @@ -89,18 +101,22 @@ in stdenv.mkDerivation ({ --add-flags "-I${llvmPackages.openmp.dev}/include" \ '' + lib.optionalString config.rocm '' --add-flags "--rocm-device-lib-path=${rocmPackages.rocm-device-libs}/amdgcn/bitcode" + '' + lib.optionalString config.cuda '' + --add-flags "--cuda-path=${cudaPackages.cudatoolkit}" ''; postPatch = '' patchShebangs . ''; -} // lib.optionalAttrs config.rocm { +} // lib.optionalAttrs (config.rocm || config.cuda) { # nixpkgs 26.05 turns the zerocallusedregs hardening flag on by default, and # clang rejects -fzero-call-used-regs for the amdgcn-amd-amdhsa target, so # every device bitcode object under src/libkernel/sscp/amdgpu fails to build. # Not an AdaptiveCpp problem -- the same source built fine on 25.11. # + # nvptx64 rejects the same flag, so the CUDA backend needs it off as well. + # # Set via optionalAttrs rather than lib.optionals so the attribute is absent - # (not just empty) without rocm, which keeps the base derivation hash stable. + # (not just empty) on the base build, which keeps its derivation hash stable. hardeningDisable = [ "zerocallusedregs" ]; }) diff --git a/default.nix b/default.nix index 916c3f5..fc8cc98 100644 --- a/default.nix +++ b/default.nix @@ -7,6 +7,13 @@ # host/OpenMP path stays cheap to build; turn on with # nix-build -A <attr> --arg rocm true , rocm ? false +# Same for the CUDA backend: +# nix-build -A <attr> --arg cuda true +# Note that nixpkgs ships the CUDA toolkit as unfree, so this additionally +# needs allowUnfree, e.g. +# nix-build -A kel.lbm.sycl-cuda --arg cuda true \\ +# --arg pkgs '(import <nixpkgs> { config.allowUnfree = true; })' +, cuda ? false }: let @@ -36,21 +43,26 @@ let ''; }; - # One builder, two instances, so the ROCm and non-ROCm toolchains differ in - # exactly one input and nothing else. - mkAdaptiveCpp = enableRocm: pkgs.callPackage .nix/adaptive-cpp.nix { + # One builder, three instances, so the GPU toolchains differ from the base + # one in exactly one input and nothing else. + mkAdaptiveCpp = backend: pkgs.callPackage .nix/adaptive-cpp.nix { inherit stdenv llvmPackages lld; config = { - rocm = enableRocm; + rocm = backend == "rocm"; + cuda = backend == "cuda"; }; }; - adaptive-cpp-base = mkAdaptiveCpp false; - adaptive-cpp-rocm = mkAdaptiveCpp true; + adaptive-cpp-base = mkAdaptiveCpp "host"; + adaptive-cpp-rocm = mkAdaptiveCpp "rocm"; + adaptive-cpp-cuda = mkAdaptiveCpp "cuda"; - # What the examples and the default build use. Flip with --arg rocm true. - adaptive-cpp = if rocm then adaptive-cpp-rocm else adaptive-cpp-base; + # What the examples and the default build use. Flip with --arg rocm/cuda true. + adaptive-cpp = + if rocm then adaptive-cpp-rocm + else if cuda then adaptive-cpp-cuda + else adaptive-cpp-base; # forstio master as of 2026-11-09. Bump together with the sha256 below. forstioRev = "97ae1a8f23074fea3784a008921c5d9fc3c1e8fb"; @@ -107,9 +119,34 @@ in rec { adaptive-cpp = adaptive-cpp-rocm; variant = "-rocm"; }; + sycl-cuda = pkgs.callPackage ./modules/sycl/.nix/derivation.nix { + inherit forstio stdenv clang-tools pname version kel; + adaptive-cpp = adaptive-cpp-cuda; + variant = "-cuda"; + }; + + # Control protocol shared by the orchestrator and the workers. + # Schemas and arithmetic only -- no transport, no GL -- so it stays + # testable natively on both sides of the browser boundary. + swarm = pkgs.callPackage ./modules/swarm/.nix/derivation.nix { + inherit forstio stdenv clang-tools pname version kel; + }; + + # Browser backend. Built by em++ rather than the nix cc-wrapper, which + # is why it takes its include roots through sconsFlags instead of + # buildInputs. + webgl = pkgs.callPackage ./modules/webgl/.nix/derivation.nix { + inherit stdenv clang-tools pname version kel; + }; }; }; + # Proof of concept work. Reachable from the top level so it actually gets + # built; it was previously orphaned and had drifted into not evaluating. + poc = import ./poc/default.nix { + inherit pkgs stdenv pname version kel; + }; + util = { lbm_ogl_renderer = pkgs.callPackage ./util/ogl_renderer/.nix/derivation.nix { inherit forstio stdenv clang-tools pname version kel; @@ -153,21 +190,6 @@ in rec { inherit kel; }; - cavity_2d = pkgs.callPackage ./examples/cavity_2d/.nix/derivation.nix { - inherit pname version stdenv forstio; - inherit kel; - }; - - meta = pkgs.callPackage ./examples/meta/.nix/derivation.nix { - inherit pname version stdenv forstio; - inherit kel; - }; - - poiseulle_2d = pkgs.callPackage ./examples/poiseulle_3d/.nix/derivation.nix { - inherit pname version stdenv forstio adaptive-cpp; - inherit kel; - }; - moving_poiseulle_particles_2d_hlbm_gpu = pkgs.callPackage ./examples/moving_poiseulle_particles_2d_hlbm_gpu/.nix/derivation.nix { inherit pname version stdenv forstio adaptive-cpp; inherit kel; @@ -249,11 +271,6 @@ in rec { inherit kel; }; - particles_gpu = pkgs.callPackage ./examples/particles_gpu/.nix/derivation.nix { - inherit pname version stdenv forstio adaptive-cpp; - inherit kel; - }; - settling_cubes_2d_ibm_gpu = pkgs.callPackage ./examples/settling_cubes_2d_ibm_gpu/.nix/derivation.nix { inherit pname version stdenv forstio adaptive-cpp; inherit kel; @@ -284,11 +301,6 @@ in rec { inherit kel; }; - heterogeneous_computing = pkgs.callPackage ./examples/heterogeneous_computing/.nix/derivation.nix { - inherit pname version stdenv forstio adaptive-cpp; - inherit kel; - }; - settling_particles = pkgs.symlinkJoin { name = "kel-lbm-settling_particles-${version}"; paths = [ @@ -311,7 +323,6 @@ in rec { paths = [ kel.lbm.core kel.lbm.sycl - ex.meta ex.poiseulle_particles_2d_bgk_gpu ex.poiseulle_particles_2d_psm_gpu ex.poiseulle_particles_2d_hlbm_gpu diff --git a/examples/cavity_2d/SConscript b/examples/cavity_2d/SConscript deleted file mode 100644 index 3fabe4b..0000000 --- a/examples/cavity_2d/SConscript +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/false - -import os -import os.path -import glob - - -Import('env') - -dir_path = Dir('.').abspath - -# Environment for base library -examples_env = env.Clone(); - -examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) -examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) - -env.sources += examples_env.sources; -env.headers += examples_env.headers; - -# Cavity2D -examples_objects = []; -examples_env.add_source_files(examples_objects, ['cavity_2d.cpp'], shared=False); -examples_env.cavity_2d = examples_env.Program('#bin/cavity_2d', [examples_objects]); - -# Set Alias -env.examples = [ - examples_env.cavity_2d -]; -env.Alias('examples', env.examples); -env.targets += ['examples']; -env.Install('$prefix/bin/', env.examples); diff --git a/examples/cavity_2d/cavity_2d.cpp b/examples/cavity_2d/cavity_2d.cpp deleted file mode 100644 index ea63745..0000000 --- a/examples/cavity_2d/cavity_2d.cpp +++ /dev/null @@ -1,456 +0,0 @@ -#include <kel/lbm/lbm.hpp> - -#include <forstio/codec/data.hpp> - -#include <iostream> -#include <fstream> -#include <cmath> - -namespace kel { -namespace lbm { -namespace sch { - -/** - * Basic distribution function - * Base type - * D - * Q - * Scalar factor - * D factor - * Q factor - */ -using T = Float32; -using D2Q5 = Descriptor<2u,5u>; -using D2Q9 = Descriptor<2u,9u>; - -template<typename Desc> -using DfCell = Cell<T, Desc, 0u, 0u, 1u>; - -template<typename Desc> -using CellInfo = Cell<UInt8, D2Q9, 1u, 0u, 0u>; - -/** - * Basic type for simulation - */ -template<typename Desc> -using CellStruct = Struct< - Member<DfCell<Desc>, "dfs">, - Member<DfCell<Desc>, "dfs_old">, - Member<CellInfo<Desc>, "info"> ->; - -/** - * Macroscopic fields, one chunk per quantity. - * - * This used to be an Array of per cell structs. The VTK writer takes a Struct - * whose members are all Chunks of the same extent, and compute_rho_u wants - * Scalar/Vector rather than a bare primitive and a FixedArray, so the layout - * moved from array-of-struct to struct-of-arrays. - */ -template<typename T, uint64_t D, uint64_t DimX, uint64_t DimY> -using MacroStruct = Struct< - Member<Chunk<Vector<T,D>, 0u, DimX, DimY>, "velocity">, - Member<Chunk<Scalar<T>, 0u, DimX, DimY>, "density"> ->; - -using CavityFieldD2Q9 = CellField<D2Q9, CellStruct<D2Q9>>; -} - - -/* -template<typename T, typename Encode> -class df_cell_view; -*/ -/** - * Minor helper for the AA-Pull Pattern, so I can use only one lattice - * - * Am I sure I want to use AA this way? - * Esoteric Twist technically reduces the needed memory access footprint - */ -/* -template<typename Desc, size_t SN, size_t DN, size_t QN, typename Encode> -class df_cell_view<sch::Cell<sch::T, Desc, SN, DN, QN>, Encode> { -public: - using Schema = sch::Cell<sch::T,Desc,SN,DN,QN>; -private: - std::array<std::decay_t<typename kel::lbm::native_data_type<sch::T>::type>*, QN> view_; -public: - df_cell_view(const std::array<std::decay_t<typename kel::lbm::native_data_type<sch::T>::type>*, QN>& view): - view_{view} - {} -}; -*/ -namespace cmpt { -struct MovingWall {}; -} - -/** - * Full-Way moving wall Bounce back, something is not right here. - * Technically it should reflect properly. - */ -template<typename Desc> -class component<sch::T, Desc, cmpt::MovingWall> { -public: - std::array<typename kel::lbm::native_data_type<sch::T>::type, Desc::D> lid_vel; - -public: - void apply( - kel::lbm::data<sch::DfCell<Desc>>& dfs - ){ - using dfi = df_info<sch::T,Desc>; - - // Technically use .copy() - /* - auto dfs_cpy = dfs; - - for(uint64_t i = 0u; i < Desc::Q; ++i){ - dfs({dfi::opposite_index.at(i)}) = dfs_cpy({i}) - 2.0 * dfi::weights[i] * 1.0 * ( lid_vel[0] * dfi::directions[i][0] + lid_vel[1] * dfi::directions[i][1]) * dfi::inv_cs2; - } - */ - } -}; -} -} - -constexpr size_t dim_size = 2; -constexpr size_t dim_x = 128; -constexpr size_t dim_y = 128; - - -template<typename Func> -void apply_for_cells(Func&& func, kel::lbm::data<kel::lbm::sch::CavityFieldD2Q9>& dat){ - for(std::size_t i = 0; i < dat.meta().at({1u}).get(); ++i){ - for(std::size_t j = 0; j < dat.meta().at({0u}).get(); ++j){ - kel::lbm::data<kel::lbm::sch::UInt64> di{i}; - kel::lbm::data<kel::lbm::sch::UInt64> dj{j}; - auto& cell_v = dat({{dj,di}}); - func(cell_v, j, i); - } - } -} - -void set_geometry(kel::lbm::data<kel::lbm::sch::CavityFieldD2Q9>& latt){ - using namespace kel::lbm; - apply_for_cells([](auto& cell, std::size_t i, std::size_t j){ - uint8_t val = 0; - if(j == 1){ - val = 2u; - } - if(i == 1 || (i+2) == dim_x || (j+2) == dim_y){ - val = 3u; - } - if(i > 1 && (i+2) < dim_x && j > 1 && (j+2) < dim_y){ - val = 1u; - } - if(i == 0 || j == 0 || (i+1) == dim_x || (j+1) == dim_y){ - val = 0u; - } - cell.template get<"info">()(0u).set(val); - }, latt); -} - -void set_initial_conditions(kel::lbm::data<kel::lbm::sch::CavityFieldD2Q9>& latt){ - using namespace kel::lbm; - - kel::lbm::data<sch::T> rho{1.0}; - { - kel::lbm::data<sch::FixedArray<sch::T,sch::D2Q9::D>> vel{{0.0,0.0}}; - auto eq = equilibrium<sch::T,sch::D2Q9>(rho, vel); - - apply_for_cells([&eq](auto& cell, std::size_t i, std::size_t j){ - (void) i; - (void) j; - auto& dfs = cell.template get<"dfs">(); - auto& dfs_old = cell.template get<"dfs_old">(); - auto info = cell.template get<"info">()(0u).get(); - for(uint64_t k = 0; k < sch::D2Q9::Q; ++k){ - dfs(k) = eq.at({k}); - dfs_old(k) = eq.at({k}); - } - }, latt); - } - - { - kel::lbm::data<sch::FixedArray<sch::T,sch::D2Q9::D>> vel{{0.1,0.0}}; - auto eq = equilibrium<sch::T,sch::D2Q9>(rho, vel); - - apply_for_cells([&eq](auto& cell, std::size_t i, std::size_t j){ - (void) i; - (void) j; - auto& dfs = cell.template get<"dfs">(); - auto& dfs_old = cell.template get<"dfs_old">(); - auto info = cell.template get<"info">()(0u).get(); - if(info == 2u){ - for(uint64_t k = 0; k < sch::D2Q9::Q; ++k){ - dfs(k) = eq.at({k}); - dfs_old(k) = eq.at({k}); - } - } - }, latt); - } -} - -void lbm_step( - kel::lbm::data<kel::lbm::sch::CavityFieldD2Q9>& latt, - bool even_step, - uint64_t time_step -){ - using namespace kel::lbm; - using dfi = df_info<sch::T,sch::D2Q9>; - - /** - * 1. Relaxation parameter \tau - */ - component<sch::T, sch::D2Q9, cmpt::BGK> coll{0.5384}; - component<sch::T, sch::D2Q9, cmpt::BounceBack> bb; - - component<sch::T, sch::D2Q9, cmpt::MovingWall> bb_lid; - bb_lid.lid_vel = {0.1,0.0}; - - auto meta = latt.meta(); - - // Collide - for(kel::lbm::data<sch::UInt64> i{0u}; i < meta.at(0u); ++i){ - for(kel::lbm::data<sch::UInt64> j{0u}; j < meta.at(1u); ++j ){ - auto& cell = latt({{i,j}}); - auto& info = cell.template get<"info">(); - - switch(info({0u}).get()){ - case 1u: { - coll.apply(latt, {{i,j}}, time_step); - break; - } - case 2u: { - auto& df = even_step ? cell.template get<"dfs_old">() : cell.template get<"dfs">(); - bb_lid.apply(df); - break; - } - case 3u: { - bb.apply(latt, {{i,j}}, time_step); - break; - } - default: - break; - } - } - } - - // Stream - for(uint64_t i = 1u; (i+1u) < latt.template get_dim_size<0>().get(); ++i){ - for(uint64_t j = 1u; (j+1u) < latt.template get_dim_size<1>().get(); ++j){ - auto& cell = latt({{i,j}}); - auto& df_new = even_step ? cell.template get<"dfs">() : cell.template get<"dfs_old">(); - auto& info_new = cell.template get<"info">(); - - if(info_new({0u}).get() > 0u && info_new({0u}).get() != 2u){ - for(uint64_t k = 0u; k < sch::D2Q9::Q; ++k){ - auto dir = dfi::directions[dfi::opposite_index[k]]; - auto& cell_dir_old = latt({{i+dir[0],j+dir[1]}}); - - auto& df_old = even_step ? cell_dir_old.template get<"dfs_old">() : cell_dir_old.template get<"dfs">(); - auto& info_old = cell_dir_old.template get<"info">(); - - if( info_old({0}).get() == 2u ){ - auto& df_old_loc = even_step ? latt({{i,j}}).template get<"dfs_old">() : latt({{i,j}}).template get<"dfs">(); - df_new({k}) = df_old_loc({dfi::opposite_index.at(k)}) - 2.0 * dfi::inv_cs2 * dfi::weights.at(k) * 1.0 * ( bb_lid.lid_vel[0] * dir[0] + bb_lid.lid_vel[1] * dir[1]); - // dfs({dfi::opposite_index.at(i)}) = dfs_cpy({i}) - 2.0 * dfi::weights[i] * 1.0 * ( lid_vel[0] * dfi::directions[i][0] + lid_vel[1] * dfi::directions[i][1]) * dfi::inv_cs2; - } else { - df_new({k}) = df_old({k}); - } - } - } - } - } -} - -int main(){ - using namespace kel::lbm; - - kel::lbm::data<sch::FixedArray<sch::UInt64,sch::D2Q9::D>> dim{{dim_x, dim_y}}; - - kel::lbm::data<sch::CavityFieldD2Q9, kel::lbm::enc::Native> lattice{dim}; - - converter<sch::T> conv{ - {0.1}, - {0.1} - }; - - // Lid driven cavity: the lid velocity and the cavity height are the - // characteristic scales. - print_lbm_meta<sch::T, sch::D2Q9>(conv, {1e-3}, {0.1}, {dim_y}); - - //auto& df_field = lattices.at(0).template get<"dfs">(); - //for(uint64_t i = 0; i < df_field.get_dim_size<0u>(); ++i){ - // lattices.at(i) = {dim_x, dim_y}; - //} - - /** - * Set meta information describing what this cell is - */ - set_geometry(lattice); - - /** - * - */ - set_initial_conditions(lattice); - - /** - * Timeloop - */ - - /** - * Print basic setup info - */ - apply_for_cells([](auto& cell, std::size_t i, std::size_t j){ - // Not needed - (void) j; - std::cout<<(static_cast<uint32_t>(cell.template get<"info">()({0}).get())); - if( (i+1) < dim_x){ - std::cout<<" "; - }else{ - std::cout<<"\n"; - } - }, lattice); - - uint64_t lattice_steps = 512000u; - bool even_step = true; - - uint64_t print_every = 256u; - uint64_t file_no = 0u; - - // Chunks carry their extent in the type, so no runtime dim argument. - kel::lbm::data<sch::MacroStruct<sch::T,sch::D2Q9::D,dim_x,dim_y>> macros; - - for(uint64_t step = 0; step < lattice_steps; ++step){ - - if(step % print_every == 0u){ - std::cout<<"\n"; - typename kel::lbm::native_data_type<sch::T>::type sum = 0.0; - - - apply_for_cells([&](auto& cell, std::size_t i, std::size_t j){ - auto& dfs = cell.template get<"dfs">(); - // compute_rho_u takes a FixedArray of populations now, while this - // example still stores them in a Cell. Same Q values, different - // schema, so copy across rather than reinterpret. - kel::lbm::data<sch::FixedArray<sch::T,sch::D2Q9::Q>> dfs_arr; - for(uint64_t q{0u}; q < sch::D2Q9::Q; ++q){ - dfs_arr.at({q}) = dfs({q}); - } - - kel::lbm::data<sch::Scalar<sch::T>> rho_d; - kel::lbm::data<sch::Vector<sch::T,sch::D2Q9::D>> vel_d; - compute_rho_u<sch::T,sch::D2Q9>(dfs_arr,rho_d,vel_d); - - const auto rho = rho_d.at({}).get(); - const typename kel::lbm::native_data_type<sch::T>::type vel[sch::D2Q9::D] = { - vel_d.at({{0u}}).get(), vel_d.at({{1u}}).get()}; - - if(i > 1 && (i+2) < dim_x && j > 1 && (j+2) < dim_y){ - sum += rho; - } - auto angle = std::atan2(vel[1],vel[0]); - double vel_mag = vel[1] * vel[1] + vel[0] * vel[0]; - - auto dir_char = [&]() -> std::string_view { - /** - * Flipped y due to print order - */ - constexpr auto pi = M_PI; - return "■"; - if(vel_mag < 1e-4){ - return "•"; - } - if(angle > 7.0 / 8.0 * pi){ - return "←"; - } - if(angle > 5.0 / 8.0 * pi){ - return "↙"; - } - if(angle > 3.0 / 8.0 * pi){ - return "↓"; - } - if(angle > 1.0 / 8.0 * pi){ - return "↘"; - } - if(angle > -1.0 / 8.0 * pi){ - return "→"; - } - if(angle > -3.0 / 8.0 * pi){ - return "↗"; - } - if(angle > -5.0 / 8.0 * pi){ - return "↑"; - } - if(angle > -7.0 / 8.0 * pi){ - return "↖"; - } - return "←"; - }(); - - std::array<uint32_t,3u> rgb_start{64,64,255}; - std::array<uint32_t,3u> rgb_stop{255,64,64}; - std::array<uint32_t,3u> rgb_middle{255,255,255}; - - - auto col_interpol = [&](){ - std::array<uint32_t, 3u> rgb_interpol = rgb_start; - double vel_mag_cut = std::min(1.0,std::max(vel_mag/(0.07*0.07),0.0)); - - if(vel_mag_cut < 0.5){ - uint32_t vel_mag_i = static_cast<uint32_t>(2.0 * vel_mag_cut * 256); - for(uint8_t i = 0u; i < 3u; ++i){ - rgb_interpol[i] = (rgb_middle[i] * vel_mag_i + (256-vel_mag_i) * rgb_start[i]) / 256; - } - }else{ - uint32_t vel_mag_i = static_cast<uint32_t>((2.0*(vel_mag_cut-0.5)) * 256); - for(uint8_t i = 0u; i < 3u; ++i){ - rgb_interpol[i] = (rgb_stop[i] * vel_mag_i + (256-vel_mag_i) * rgb_middle[i]) / 256; - } - } - return rgb_interpol; - }; - auto rgb_interpol = col_interpol(); - - std::cout<<"\x1b[38;2;"<<rgb_interpol[0]<<";"<<rgb_interpol[1]<<";"<<rgb_interpol[2]<<"m"<<dir_char; - if( (i+1) < dim_x){ - std::cout<<" "; - }else{ - std::cout<<"\n"; - } - }, lattice); - std::cout<<"\x1b[38;2;255;255;255m"; - - /// std::cout<<"Average rho: "<<(sum / ((dim_x-4)*(dim_y-4)))<<std::endl; - std::cout.flush(); - - apply_for_cells([&](auto& cell, std::size_t i, std::size_t j){ - auto dfs = even_step ? cell.template get<"dfs_old">() : cell.template get<"dfs">(); - - kel::lbm::data<sch::FixedArray<sch::T,sch::D2Q9::Q>> dfs_arr; - for(uint64_t q{0u}; q < sch::D2Q9::Q; ++q){ - dfs_arr.at({q}) = dfs({q}); - } - - auto& rho = macros.template get<"density">().at({{i,j}}); - auto& vel = macros.template get<"velocity">().at({{i,j}}); - compute_rho_u<sch::T,sch::D2Q9>(dfs_arr,rho,vel); - }, lattice); - - write_vtk_file("tmp", "cavity_2d", file_no, macros); - - ++file_no; - } - - lbm_step(lattice, even_step, step); - - even_step = not even_step; - } - - /** - * Flush cout - */ - std::cout<<"\n\n"; - std::cout.flush(); - return 0; -} diff --git a/examples/cavity_2d_gpu/.nix/derivation.nix b/examples/cavity_2d_gpu/.nix/derivation.nix index 8724d96..3b210a3 100644 --- a/examples/cavity_2d_gpu/.nix/derivation.nix +++ b/examples/cavity_2d_gpu/.nix/derivation.nix @@ -23,13 +23,16 @@ stdenv.mkDerivation { buildInputs = [ forstio.core + forstio.io forstio.async forstio.codec forstio.codec-unit forstio.remote + forstio.remote-filesystem forstio.codec-json adaptive-cpp kel.lbm.core + kel.lbm.sycl # kel-lbm.sycl ]; diff --git a/examples/cavity_2d_gpu/SConstruct b/examples/cavity_2d_gpu/SConstruct index fe206e1..7882265 100644 --- a/examples/cavity_2d_gpu/SConstruct +++ b/examples/cavity_2d_gpu/SConstruct @@ -59,6 +59,8 @@ env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], ], LIBS=[ 'forstio-core', + 'forstio-async', + 'forstio-io', 'acpp-rt', 'omp' ] diff --git a/examples/cavity_2d_gpu/cavity_2d_gpu.cpp b/examples/cavity_2d_gpu/cavity_2d_gpu.cpp index 55225d1..6b71db3 100644 --- a/examples/cavity_2d_gpu/cavity_2d_gpu.cpp +++ b/examples/cavity_2d_gpu/cavity_2d_gpu.cpp @@ -1,384 +1,464 @@ +#include <kel/lbm/sycl/lbm.hpp> #include <kel/lbm/lbm.hpp> -#include <forstio/codec/data.hpp> -#include <AdaptiveCpp/sycl/sycl.hpp> - -#include <iostream> -#include <fstream> -#include <vector> -#include <chrono> -#include <cmath> +#include <forstio/io/io.hpp> +#include <forstio/remote/filesystem/easy.hpp> +#include <forstio/codec/json/json.hpp> +#include <forstio/codec/simple.hpp> +#include <forstio/codec/args.hpp> namespace kel { namespace lbm { -namespace sch { -/** - * Basic distribution function - * Base type - * D - * Q - * Scalar factor - * D factor - * Q factor - */ -using T = Float32; -using D2Q9 = Descriptor<2u,9u>; +// Square cavity, as this case has always used. +constexpr uint64_t dim_x = 128ul; +constexpr uint64_t dim_y = 128ul; -template<typename Desc> -using DfCell = Cell<T, Desc, 0u, 0u, 1u>; +// Lid velocity, tangential along +x on the top row. +constexpr double lid_velocity = 0.1; -template<typename Desc> -using CellInfo = Cell<UInt8, Desc, 1u, 0u, 0u>; - -/** - * Basic type for simulation - */ -template<typename Desc> -using CellStruct = CellFieldStruct< - Desc, - Struct< - Member<CellField<Desc,DfCell<Desc>>, "dfs">, - Member<CellField<Desc,DfCell<Desc>>, "dfs_old">, - Member<CellField<Desc,CellInfo<Desc>>, "info"> - > ->; +namespace sch { -template<typename T, uint64_t D> -using MacroStruct = Struct< - Member<FixedArray<T,D>, "velocity">, - Member<T, "pressure"> ->; +using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y>; -using CavityFieldD2Q9 = CellStruct<D2Q9>; -} +template<typename T, typename Desc> +using DfChunk = Chunk<FixedArray<T,Desc::Q>, 1u, dim_x, dim_y>; +template<typename T, typename Desc> +using ScalarChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; -/* -template<typename T, typename Encode> -class df_cell_view; -*/ -/** - * Minor helper for the AA-Pull Pattern, so I can use only one lattice - * - * Am I sure I want to use AA this way? - * Esoteric Twist technically reduces the needed memory access footprint - */ -/* -template<typename Desc, size_t SN, size_t DN, size_t QN, typename Encode> -class df_cell_view<sch::Cell<sch::T, Desc, SN, DN, QN>, Encode> { -public: - using Schema = sch::Cell<sch::T,Desc,SN,DN,QN>; -private: - std::array<std::decay_t<typename kel::lbm::native_data_type<sch::T>::type>*, QN> view_; -public: - df_cell_view(const std::array<std::decay_t<typename kel::lbm::native_data_type<sch::T>::type>*, QN>& view): - view_{view} - {} -}; -*/ -namespace cmpt { -struct MovingWall {}; -} +template<typename T, typename Desc> +using VectorChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; -/** - * Full-Way moving wall Bounce back, something is not right here. - * Technically it should reflect properly. - */ -template<typename Desc> -class component<sch::T, Desc, cmpt::MovingWall> { -public: - std::array<typename kel::lbm::native_data_type<sch::T>::type, Desc::D> lid_vel; +template<typename T, typename Desc> +using ChunkStruct = Struct< + Member<InfoChunk, "info">, + Member<DfChunk<T,Desc>, "dfs">, + Member<DfChunk<T,Desc>, "dfs_old"> +>; -public: - void apply( - kel::lbm::data<sch::DfCell<Desc>>& dfs - ){ - using dfi = df_info<sch::T,Desc>; +template<typename T, typename Desc> +using VelChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; - // Technically use .copy() - /* - auto dfs_cpy = dfs; +template<typename T> +using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; - for(uint64_t i = 0u; i < Desc::Q; ++i){ - dfs({dfi::opposite_index.at(i)}) = dfs_cpy({i}) - 2.0 * dfi::weights[i] * 1.0 * ( lid_vel[0] * dfi::directions[i][0] + lid_vel[1] * dfi::directions[i][1]) * dfi::inv_cs2; - } - */ - } -}; +template<typename T, typename Desc> +using MacroStruct = Struct< + Member<VelChunk<T,Desc>, "velocity">, + Member<RhoChunk<T>, "density"> +>; -constexpr size_t dim_x = 128; -constexpr size_t dim_y = 128; +//template<typename T, typename Desc> +//using ParticleArray = Array< +// Particle<T,Desc::D> +//>; +} -template<typename Desc> -void set_geometry( - kel::lbm::data<sch::CellInfo<Desc>>* info_field +template<typename T, typename Desc> +kel::lbm::error_or<void> setup_initial_conditions( + kel::lbm::data<sch::ChunkStruct<T,Desc>>& fields, + kel::lbm::data<sch::MacroStruct<T,Desc>>& macros ){ - using namespace kel::lbm; - using namespace acpp; - kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> meta{{dim_x,dim_y}}; - /** - * Set ghost - */ - iterate_over([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,2u>>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - - info({0u}).set(0u); - - }, {{0u,0u}}, meta); - - /** - * Set wall - */ - iterate_over([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,2u>>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - - info({0u}).set(1u); - - }, {{0u,0u}}, meta, {{1u,1u}}); - - /** - * Set fluid - */ - iterate_over([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,2u>>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - - info({0u}).set(2u); - - }, {{0u,0u}}, meta, {{2u,2u}}); - - /** - * Set top lid - */ - iterate_over([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,2u>>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& info = info_field[i]; - info({0u}).set(3u); + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u}} + ); + + // The lid: the top row, moving tangentially. Corners stay plain walls. + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(5u); + }, + {{0u,dim_y-1u}}, + {{dim_x,dim_y}}, + {{1u,0u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& vel_f = macros.template get<"velocity">(); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& vel = vel_f.at(index); + auto eq = equilibrium<T,Desc>(rho,vel); - }, {{0u,1u}}, {{meta.at({0u}), 2u}}, {{2u,0u}}); -} + df = eq; + }, + {},// 0-index + df_f.get_dims() + ); -template<typename Desc> -void set_initial_conditions( - kel::lbm::data<sch::CellInfo<Desc>>* info_field, - kel::lbm::data<sch::DfCell<Desc>>* dfs_field, - kel::lbm::data<sch::DfCell<Desc>>* dfs_old_field -){ - using namespace kel::lbm; - using namespace acpp; + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& vel = vel_f.at(index); + if(info_f.at(index).get() == 2u){ + vel.at({{0u}}) = 0.0; + } + auto eq = equilibrium<T,Desc>(rho,vel); - kel::lbm::data<sch::T> rho{1.0}; - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>> vel{{0.0,0.0}}; - auto eq = equilibrium<sch::T,Desc>(rho, vel); - kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> meta{{dim_x,dim_y}}; + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u}} + ); + + iterator<Desc::D>::apply( + [&](auto& index){ + kel::lbm::data<sch::Vector<T,Desc::D>> middle, ind_vec; + middle.at({{0u}}) = dim_x * 0.25; + middle.at({{1u}}) = dim_y * 0.5; - /** - * Set distribution - */ - iterate_over([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>>& index){ - size_t i = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto& dfs = dfs_field[i]; - auto& dfs_old = dfs_old_field[i]; + ind_vec.at({{0u}}) = index.at({{0u}}).template cast_to<T>(); + ind_vec.at({{1u}}) = index.at({{1u}}).template cast_to<T>(); - for(kel::lbm::data<sch::UInt64> k = 0; k < kel::lbm::data<sch::UInt64>{Desc::Q}; ++k){ - dfs(k) = eq.at(k); - dfs_old(k) = eq.at(k); - } + auto dist = middle - ind_vec; + auto dist_2 = kel::lbm::math::dot(dist,dist); + if(dist_2.at({}).get() < dim_y*dim_y*0.01){ + info_f.at(index) = 1u; + } + }, + {},// 0-index + df_f.get_dims() + ); - }, {{0u,0u}}, meta); + return kel::lbm::make_void(); } template<typename T, typename Desc> -void lbm_step( - kel::lbm::data<sch::CellInfo<Desc>>* info_r, - kel::lbm::data<sch::DfCell<Desc>>* dfs_r, - kel::lbm::data<sch::DfCell<Desc>>* dfs_r_old, - bool is_even, - uint64_t time_step, - acpp::sycl::queue& sycl_q +kel::lbm::error_or<void> step( + kel::lbm::data<sch::Ptr<sch::ChunkStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& fields, + kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, + kel::lbm::data<sch::UInt64> t_i, + device& dev ){ - using namespace kel::lbm; - using namespace acpp; - - using dfi = df_info<T,Desc>; + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); - //component<T, Desc, cmpt::BGK> coll{0.59}; - constexpr kel::lbm::data<T> frequency{1.0 / 0.59}; - //component<T, Desc, cmpt::BounceBack> bb; - component<T, Desc, cmpt::MovingWall> bb_lid; - bb_lid.lid_vel = {0.1, 0.0}; + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + // Need nicer things to handle the flow. I see improvement here + component<T,Desc,cmpt::BGK, encode::Sycl<kel::lbm::enc::Native>> collision{0.8}; + component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; + component<T,Desc,cmpt::AntiBounceBack<0u>,encode::Sycl<kel::lbm::enc::Native>> abb; - // Submit collision kernel - sycl_q.submit([&](sycl::handler& cgh) { - // Accessor for latt with read/write + kel::lbm::data<sch::Scalar<T>> rho_b; + rho_b.at({}) = 1.0; + kel::lbm::data<sch::Vector<T,Desc::D>> vel_b; + vel_b.at({{0u}}) = 0.015; - cgh.parallel_for( - sycl::range<2>{dim_x, dim_y}, [=](sycl::id<2> idx) { - size_t i = idx[0]; - size_t j = idx[1]; + component<T,Desc,cmpt::Equilibrium,encode::Sycl<kel::lbm::enc::Native>> equi{rho_b,vel_b}; - size_t acc_id = i * dim_x + j; - - // Read cell info - auto& info = info_r[acc_id]; - - switch (info({0u}).get()) { - case 1u: { - // bb.apply(latt_acc, {i, j}, time_step); - - auto& dfs_old = is_even ? dfs_r_old[acc_id] : dfs_r[acc_id]; - auto df_cpy = dfs_old.copy(); - - for(uint64_t i = 1u; i < Desc::Q; ++i){ - dfs_old({i}) = df_cpy({dfi::opposite_index.at(i)}); + component<T,Desc,cmpt::ZouHeHorizontal<true>,encode::Sycl<kel::lbm::enc::Native>> flow_in{ + [&](){ + uint64_t target_t_i = 32u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.0002 / target_t_i) * t_i.get(); } + return 1.0002; + }() + }; + component<T,Desc,cmpt::ZouHeHorizontal<false>,encode::Sycl<kel::lbm::enc::Native>> flow_out{1.0}; - break; - } - case 2u: { - // coll.apply(latt_acc, {i, j}, time_step); - auto& dfs_old = is_even ? dfs_r_old[acc_id] : dfs_r[acc_id]; - kel::lbm::data<T> rho; - kel::lbm::data<sch::FixedArray<T,Desc::D>> vel; - compute_rho_u<T,Desc>(dfs_old,rho,vel); - auto eq = equilibrium<T,Desc>(rho,vel); + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } - for(uint64_t i = 0u; i < Desc::Q; ++i){ - dfs_old({i}) = dfs_old({i}) + frequency * (eq.at(i) - dfs_old({i})); - } + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: break; - } - default: - // Do nothing + case 1u: + bb.apply(fields,index,t_i); break; - } - }); - }); - - // Submit streaming kernel - sycl_q.submit([&](sycl::handler& cgh) { - - cgh.parallel_for( - sycl::range<2>{dim_x - 2, dim_y - 2}, [=](sycl::id<2> idx) { - size_t i = idx[0] + 1; // avoid boundary - size_t j = idx[1] + 1; - - size_t acc_id = i * dim_x + j; - - auto& dfs_new = is_even ? dfs_r[acc_id] : dfs_r_old[acc_id]; - auto& info = info_r[acc_id]; + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 5u: { + /** + * Moving wall: bounce back plus the tangential momentum the + * lid injects. + * + * f_i = f_opp + 2 w_i rho_w (e_i . u_w) / cs^2 + * + * written from f_opp because that is the direction the + * library's BounceBack writes in. There is no component for a + * y normal moving wall, and the original example had this + * commented out, so it is implemented here. + */ + using dfi = df_info<T,Desc>; + const bool is_even = ((t_i.get() % 2u) == 0u); + auto& dfs_f = (is_even) + ? fields.template get<"dfs_old">() + : fields.template get<"dfs">(); + auto& dfs = dfs_f.at(index); + auto df_cpy = dfs; - if (info({0u}).get() > 0u && info({0u}).get() != 3u) { - for (uint64_t k = 0u; k < Desc::Q; ++k) { - auto dir = dfi::directions[dfi::opposite_index[k]]; - // auto& cell_dir_old = latt_acc({i + dir[0], j + dir[1]}); - // - size_t acc_old_id = (i+dir[0]) * dim_x + (j+dir[1]); + for(uint64_t i = 0u; i < Desc::Q; ++i){ + const auto e_dot_u = + static_cast<typename kel::lbm::native_data_type<T>::type>( + dfi::directions[i][0]) * lid_velocity; - auto& dfs_old = is_even ? dfs_r_old[acc_old_id] : dfs_r[acc_old_id]; - auto& info_old = info_r[acc_old_id]; - - if (info_old({0}).get() == 3u) { - auto& dfs_old_loc = is_even ? dfs_r_old[acc_id] : dfs_r[acc_id]; - dfs_new({k}) = dfs_old_loc({dfi::opposite_index.at(k)}) - 2.0 * dfi::inv_cs2 * dfi::weights.at(k) * 1.0 * (bb_lid.lid_vel[0] * dir[0] + bb_lid.lid_vel[1] * dir[1]); - } else { - dfs_new({k}) = dfs_old({k}); + dfs.at({i}) = df_cpy.at({dfi::opposite_index.at(i)}) + + kel::lbm::data<T>{ + static_cast<typename kel::lbm::native_data_type<T>::type>( + 2.0 * dfi::weights[i] * dfi::inv_cs2 * e_dot_u)}; } - } + } break; + default: + break; } }); - }); + }).wait(); + + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ + + return kel::lbm::make_void(); } } } -int main(){ +template<typename T, typename Desc> +kel::lbm::error_or<void> lbm_main(int argc, char** argv){ using namespace kel::lbm; - using namespace acpp; - kel::lbm::data<sch::FixedArray<sch::UInt64,sch::D2Q9::D>> meta{{dim_x, dim_y}}; - constexpr size_t dim_size = dim_x * dim_y; - - converter<sch::T> conv{ - {0.1}, - {0.1} - }; + /* + auto eo_lbm_args = setup_lbm_env<sch::Args<sch::Struct<>,sch::Tuple<>>>(argc,argv); + if(eo_lbm_args.is_error()){ + return std::move(eo_lbm_args.get_error()); + } + auto& lbm_args = eo_lbm_args.get_value(); + */ - // Lid driven cavity: the lid velocity and the cavity height are the - // characteristic scales. - print_lbm_meta<sch::T, sch::D2Q9>(conv, {1e-3}, {0.1}, {dim_y}); + using dfi = df_info<T,Desc>; auto eo_lbm_dir = output_directory(); if(eo_lbm_dir.is_error()){ - return -1; + return std::move(eo_lbm_dir.get_error()); } auto& lbm_dir = eo_lbm_dir.get_value(); - auto out_dir = lbm_dir / "cavity_gpu_2d"; - sycl::queue sycl_q{sycl::default_selector_v, sycl::property::queue::in_order{}}; + auto out_dir = lbm_dir / "cavity_2d_gpu"; - constexpr size_t num_cells = dim_x * dim_y; + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not create output directory"); + } + } - // Allocate USM shared memory for your main data - auto* info = sycl::malloc_shared<kel::lbm::data<sch::CellInfo<sch::D2Q9>>>(num_cells, sycl_q); - auto* dfs = sycl::malloc_shared<kel::lbm::data<sch::DfCell<sch::D2Q9>>>(num_cells, sycl_q); - auto* dfs_old = sycl::malloc_shared<kel::lbm::data<sch::DfCell<sch::D2Q9>>>(num_cells, sycl_q); + converter<T> conv { + // delta_x + {1.0}, + // delta_t + {1.0} + }; + + print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); - /** - * Set meta information describing what this cell is - */ - set_geometry(info); + // kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> meta{{dim_x,dim_y}}; + auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); + auto lbm_macro_data_ptr = kel::lbm::heap<kel::lbm::data<sch::MacroStruct<T,Desc>>>(); + + std::cout<<"Estimated Bytes: "<<memory_estimate<sch::ChunkStruct<T,Desc>,sch::MacroStruct<T,Desc>>().get()<<std::endl; - /** - * - */ - set_initial_conditions(info,dfs,dfs_old); + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error<kel::lbm::err::critical>( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; - /** - * Timeloop - */ - uint64_t lattice_steps = 512000u; - bool even_step = true; + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); - uint64_t print_every = 256u; - uint64_t file_no = 0u; + device dev; - kel::lbm::data<sch::Array<sch::MacroStruct<sch::T,sch::D2Q9::D>,sch::D2Q9::D>> macros{meta}; + auto& sycl_q = dev.get_handle(); - std::cout<<"Start"<<std::endl; + sycl_q.wait(); + { + auto eov = setup_initial_conditions<T,Desc>(*lbm_data_ptr,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"initial_state",0u,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } - auto start = std::chrono::steady_clock::now(); + kel::lbm::data<sch::ChunkStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_data{sycl_q}; + kel::lbm::data<sch::MacroStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_macro_data{sycl_q}; + sycl_q.wait(); + + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } sycl_q.wait(); - for(uint64_t i = 0u; i < lattice_steps; ++i){ - lbm_step<sch::T,sch::D2Q9>(info, dfs, dfs_old, even_step, i, sycl_q); - if(i % 1024u == 0u){ - sycl_q.wait(); - iterate_over([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::D2Q9::D>>& index){ - size_t j = index.at({0u}).get() * dim_x + index.at({1u}).get(); - auto dfs_field = even_step ? dfs_old : dfs; + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + kel::lbm::data<sch::UInt64> time_steps{16u*4096ul}; + + auto& info_f = lsd_view.template get<"info">(); + + for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ + // BC + Collision + { + auto eov = step<T,Desc>(lsd_view,lsdm_view,i,dev); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + /* + if(i.get() % 128u == 0u){ + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + */ + /* + if(i.get() % 32u == 0u){ + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_csv_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + */ + // Stream + sycl_q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::Stream,encode::Sycl<kel::lbm::enc::Native>> stream; - auto& rho = macros.at(index).template get<"pressure">(); - auto& vel = macros.at(index).template get<"velocity">(); - compute_rho_u<sch::T,sch::D2Q9>(dfs_field[j],rho,vel); - }, {{0u,0u}}, meta); - std::string vtk_f_name{"tmp/cavity_2d_gpu_"}; - vtk_f_name += std::to_string(i) + ".vtk"; - write_vtk_file(vtk_f_name, macros); + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(lsd_view,index,i); + } + }); + }).wait(); + wait.poll(); + if(print_status){ + std::cout<<"Status: "<<i.get()<<" of "<<time_steps.get()<<" - "<<(i.template cast_to<sch::Float64>().get() * 100 / time_steps.get())<<"%"<<std::endl; + print_status = false; } + print_progress_bar(i.get(), time_steps.get()-1u); + } - even_step = not even_step; + // After Loop + sycl_q.wait(); + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"m",time_steps.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_csv_file(out_dir,"m",time_steps.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } } - auto stop = std::chrono::steady_clock::now(); - std::cout<<std::format("{:%H:%M:%S}",(stop-start))<<std::endl; sycl_q.wait(); - sycl::free(info,sycl_q); - sycl::free(dfs,sycl_q); - sycl::free(dfs_old,sycl_q); + return kel::lbm::make_void(); +} + +using FloatT = kel::lbm::sch::Float32; + +int main(int argc, char** argv){ + auto eov = lbm_main<FloatT,kel::lbm::sch::D2Q9>(argc, argv); + if(eov.is_error()){ + auto& err = eov.get_error(); + std::cerr<<"[Error] "<<err.get_category(); + auto err_msg = err.get_message(); + if(err_msg.size() > 0u){ + std::cerr<<" - "<<err_msg; + } + std::cerr<<std::endl; + return err.get_id(); + } return 0; -}_ +} diff --git a/examples/heterogeneous_computing/.nix/derivation.nix b/examples/heterogeneous_computing/.nix/derivation.nix deleted file mode 100644 index bf824bf..0000000 --- a/examples/heterogeneous_computing/.nix/derivation.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ lib -, stdenv -, scons -, clang-tools -, forstio -, pname -, version -, adaptive-cpp -, kel -}: - -stdenv.mkDerivation { - pname = pname + "-examples-" + "heterogeneous_computing"; - inherit version; - src = ./..; - - nativeBuildInputs = [ - scons - clang-tools - ]; - - buildInputs = [ - forstio.core - forstio.async - forstio.io - forstio.io_codec - forstio.codec - forstio.codec-unit - forstio.codec-json - forstio.remote - forstio.remote-filesystem - forstio.remote-io - forstio.remote-sycl - kel.lbm.core - adaptive-cpp - ]; - - preferLocalBuild = true; - - outputs = [ "out" "dev" ]; -} diff --git a/examples/heterogeneous_computing/README.md b/examples/heterogeneous_computing/README.md deleted file mode 100644 index 16463e6..0000000 --- a/examples/heterogeneous_computing/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Heterogeneous - -This is more intended as a thinking ground for a solution to LBM integrations for distributed computing. -In the end MPI does it correctly in a sense I would almost ask to separate in blocks. -But I want to learn, so no MPI. -Or rather MPI abstracted away. -MPI also has some discovery issues in edge cases where the network is a bit more complex. diff --git a/examples/heterogeneous_computing/SConscript b/examples/heterogeneous_computing/SConscript deleted file mode 100644 index 226185b..0000000 --- a/examples/heterogeneous_computing/SConscript +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/false - -import os -import os.path -import glob - - -Import('env') - -dir_path = Dir('.').abspath - -# Environment for base library -examples_env = env.Clone(); -examples_env['CXX'] = 'syclcc-clang'; -examples_env['CXXFLAGS'] += ['-O3']; - -examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) -examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) - -env.sources += examples_env.sources; -env.headers += examples_env.headers; - -# Cavity2D -examples_objects = []; -examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); -examples_env.heterogeneous_computing = examples_env.Program('#bin/heterogeneous_computing', [examples_objects]); - -# Set Alias -env.examples = [ - examples_env.heterogeneous_computing -]; -env.Alias('examples', env.examples); -env.targets += ['examples']; -env.Install('$prefix/bin/', env.examples); diff --git a/examples/heterogeneous_computing/SConstruct b/examples/heterogeneous_computing/SConstruct deleted file mode 100644 index fe206e1..0000000 --- a/examples/heterogeneous_computing/SConstruct +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import os.path -import glob -import re - - -if sys.version_info < (3,): - def isbasestring(s): - return isinstance(s,basestring) -else: - def isbasestring(s): - return isinstance(s, (str,bytes)) - -def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): - - if isbasestring(filetype): - dir_path = self.Dir('.').abspath - filetype = sorted(glob.glob(dir_path+"/"+filetype)) - - for path in filetype: - target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) - if shared: - target_name+='.os' - sources.append( self.SharedObject( target=target_name, source=path ) ) - else: - target_name+='.o' - sources.append( self.StaticObject( target=target_name, source=path ) ) - pass - -def isAbsolutePath(key, dirname, env): - assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) - -env_vars = Variables( - args=ARGUMENTS -) - -env_vars.Add('prefix', - help='Installation target location of build results and headers', - default='/usr/local/', - validator=isAbsolutePath -) - -env_vars.Add('build_examples', - help='If examples should be built', - default="true" -) - -env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], - CPPDEFINES=['SAW_UNIX'], - CXXFLAGS=[ - '-std=c++20', - '-g', - '-Wall', - '-Wextra', - '-isystem', 'AdaptiveCpp' - ], - LIBS=[ - 'forstio-core', - 'acpp-rt', - 'omp' - ] -); -env.__class__.add_source_files = add_kel_source_files -env.Tool('compilation_db'); -env.cdb = env.CompilationDatabase('compile_commands.json'); - -env.objects = []; -env.sources = []; -env.headers = []; -env.targets = []; - -Export('env') -SConscript('SConscript') - -env.Alias('cdb', env.cdb); -env.Alias('all', [env.targets]); -env.Default('all'); - -env.Alias('install', '$prefix') diff --git a/examples/heterogeneous_computing/sim.cpp b/examples/heterogeneous_computing/sim.cpp deleted file mode 100644 index 61058be..0000000 --- a/examples/heterogeneous_computing/sim.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include <forstio/error.hpp> -#include <iostream> - -#include <kel/lbm/lbm.hpp> - -namespace kel { -namespace lbm { -namespace sch { - -/** - * struct lbm_data { - * std::array<std::array<float,9>, 64u*64u> dfs; - * std::array<uint8_t, 64u*64u> info; - * }; - * - * which leads to the form - * - * template<uint64_t Dim, uint64_t Size> - * struct lbm_data { - * std::array<std::array<float,9>, Size*Size> dfs; - * std::array<uint8_t, Size*Size> info; - * }; - * - * which transferred into sycl requires us to go to - * - * template<uint64_t Dim, uint64_t Size> - * struct lbm_sycl_data { - * std::array<float,9>* dfs; - * uint8_t* info; - * }; - * - * in data form on host - * - * template<uint64_t Dim, uint64_t Size> - * using LbmData = Struct< - * Member<FixedArray<FixedArray<Float32,9u>, Size*Size>, "dfs">, - * Member<FixedArray<UInt8, Size*Size>, "info"> - * >; - * - * If we specialize the encode::Sycl<Encoding> data type, then we get - * With a helper class we can copy single values back and forth and the whole block is guaranteed - * to be allocated. And this one can be dynamic while the host definition might be compile time. - * - * template<...> - * class data<LbmData<...>,encode::Sycl<Encoding>> final { - * kel::lbm::data<sch::FixedArray<sch::UInt64,Dim>> meta; - * kel::lbm::data<FixedArray<Float32u,9>>* dfs; - * kel::lbm::data<UInt8>* info; - * }; - */ - -template<typename T, typename Desc> -using CellStruct = Struct< - Member<Chunk<FixedArray<T,Desc::Q>,Desc::D, "dfs">, - Member<FixedArray<T,Desc::Q>, "dfs_old">, - Member<UInt8, "info"> ->; - -} -template<typename T, typename Desc> -kel::lbm::error_or<void> simulate(int argc, char** argv, - const kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>>& meta -{ - constexpr auto cell_size = sizeof(kel::lbm::data<sch::CellStruct<T,Desc>>); - - auto lbm_data = kel::lbm::heap<kel::lbm::data<sch::Chunk<CellStruct<>,Desc::D, 64u>>>{meta}; - - return kel::lbm::make_void(); -} - -} - -kel::lbm::error_or<void> lbm_main(int argc, char** argv){ - using namespace lbm; - return simulate<sch::Float64,sch::D2Q9>(argc, argv); -} -} - -int main(int argc, char** argv){ - auto eov = kel::lbm_main(argc, argv); - if(eov.is_error()){ - auto& err = eov.get_error(); - auto err_msg = err.get_message(); - std::cerr<<"[Error]: "<<err.get_category(); - - if(not err_msg.empty()){ - std::cerr<<" - "<<err_msg; - } - std::cerr<<std::endl; - - return err.get_id(); - } - - return 0; -} diff --git a/examples/meta/.nix/derivation.nix b/examples/meta/.nix/derivation.nix deleted file mode 100644 index 26706e6..0000000 --- a/examples/meta/.nix/derivation.nix +++ /dev/null @@ -1,33 +0,0 @@ -{ lib -, stdenv -, scons -, clang-tools -, forstio -, pname -, version -, kel -}: - -stdenv.mkDerivation { - pname = pname + "-examples-" + "meta_2d"; - inherit version; - src = ./..; - - nativeBuildInputs = [ - scons - clang-tools - ]; - - buildInputs = [ - forstio.core - forstio.async - forstio.codec - forstio.codec-unit - forstio.codec-json - kel.lbm.core - ]; - - preferLocalBuild = true; - - outputs = [ "out" "dev" ]; -} diff --git a/examples/meta/SConscript b/examples/meta/SConscript deleted file mode 100644 index 066a741..0000000 --- a/examples/meta/SConscript +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/false - -import os -import os.path -import glob - - -Import('env') - -dir_path = Dir('.').abspath - -# Environment for base library -examples_env = env.Clone(); - -examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) -examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) - -env.sources += examples_env.sources; -env.headers += examples_env.headers; - -# Cavity2D -examples_objects = []; -examples_env.add_source_files(examples_objects, ['meta.cpp'], shared=False); -examples_env.meta_2d = examples_env.Program('#bin/meta', [examples_objects]); - -# Set Alias -env.examples = [ - examples_env.meta_2d -]; -env.Alias('examples', env.examples); -env.targets += ['examples']; -env.Install('$prefix/bin/', env.examples); diff --git a/examples/meta/SConstruct b/examples/meta/SConstruct deleted file mode 100644 index a7201cb..0000000 --- a/examples/meta/SConstruct +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import os.path -import glob -import re - - -if sys.version_info < (3,): - def isbasestring(s): - return isinstance(s,basestring) -else: - def isbasestring(s): - return isinstance(s, (str,bytes)) - -def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): - - if isbasestring(filetype): - dir_path = self.Dir('.').abspath - filetype = sorted(glob.glob(dir_path+"/"+filetype)) - - for path in filetype: - target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) - if shared: - target_name+='.os' - sources.append( self.SharedObject( target=target_name, source=path ) ) - else: - target_name+='.o' - sources.append( self.StaticObject( target=target_name, source=path ) ) - pass - -def isAbsolutePath(key, dirname, env): - assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) - -env_vars = Variables( - args=ARGUMENTS -) - -env_vars.Add('prefix', - help='Installation target location of build results and headers', - default='/usr/local/', - validator=isAbsolutePath -) - -env_vars.Add('build_examples', - help='If examples should be built', - default="true" -) - -env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], - CPPDEFINES=['SAW_UNIX'], - CXXFLAGS=[ - '-std=c++20', - '-g', - '-Wall', - '-Wextra' - ], - LIBS=[ - 'forstio-core' - ] -); -env.__class__.add_source_files = add_kel_source_files -env.Tool('compilation_db'); -env.cdb = env.CompilationDatabase('compile_commands.json'); - -env.objects = []; -env.sources = []; -env.headers = []; -env.targets = []; - -Export('env') -SConscript('SConscript') - -env.Alias('cdb', env.cdb); -env.Alias('all', [env.targets]); -env.Default('all'); - -env.Alias('install', '$prefix') diff --git a/examples/meta/meta.cpp b/examples/meta/meta.cpp deleted file mode 100644 index 3e573e5..0000000 --- a/examples/meta/meta.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include <kel/lbm/lbm.hpp> - -#include <iostream> - -namespace kel { -namespace lbm { -namespace sch { - -using MetaArgsStruct = Struct< - Member<Float64, "dx">, - Member<Float64, "dt">, - Member<Float64, "kin_vis">, - Member<Float64, "char_vel">, - Member<Float64, "char_len"> ->; - -using MetaArgs = Args< - MetaArgsStruct, - Tuple<UInt64,UInt64> ->; - -using Prec = Float64; -} -} -} - -template<typename T, typename Desc> -kel::lbm::error_or<void> meta_lbm_info(const kel::lbm::data<kel::lbm::sch::MetaArgsStruct>& args_str){ - using namespace kel::lbm; - converter<T> conv{ - as_unit<typename unit_division<sch::SiMeter<T>, sch::LbmMeter<T>>::Schema>( - args_str.template get<"dx">()), - as_unit<typename unit_division<sch::SiSecond<T>, sch::LbmSecond<T>>::Schema>( - args_str.template get<"dt">()) - }; - print_lbm_meta<T,Desc>( - conv, - as_unit<sch::SiKinematicViscosity<T>>(args_str.template get<"kin_vis">()), - as_unit<sch::SiVelocity<T>>(args_str.template get<"char_vel">()), - as_unit<sch::SiMeter<T>>(args_str.template get<"char_len">())); - - return kel::lbm::make_void(); -} - - -kel::lbm::error_or<void> meta_main(int argc, char** argv){ - using namespace kel::lbm; - - kel::lbm::data<sch::MetaArgs> args_dat; - auto& args_str = args_dat.template get<"args">(); - - // Defaults for anything the caller leaves out. - args_str.template get<"dx">().set(1); - args_str.template get<"dt">().set(1); - - { - auto eov = parse_args(args_dat, argc, argv); - if(eov.is_error()){ - return eov; - } - } - - auto& args_pos = args_dat.template get<"positionals">(); - - switch(args_pos.template get<0u>().get()){ - case 1u: - { - switch(args_pos.template get<1u>().get()){ - case 3u: - meta_lbm_info<sch::Prec,sch::Descriptor<1u,3u>>(args_str); - break; - default: - return kel::lbm::make_error<kel::lbm::err::critical>("Second positional needs to be a valid momentum set"); - } - } - break; - case 2u: - { - switch(args_pos.template get<1u>().get()){ - case 5u: - meta_lbm_info<sch::Prec,sch::Descriptor<2u,5u>>(args_str); - break; - case 9u: - meta_lbm_info<sch::Prec,sch::Descriptor<2u,9u>>(args_str); - break; - default: - return kel::lbm::make_error<kel::lbm::err::critical>("Second positional needs to be a valid momentum set"); - } - } - break; - case 3u: - { - switch(args_pos.template get<1u>().get()){ - case 7u: - return kel::lbm::make_error<kel::lbm::err::not_implemented>("D3Q7 Velocity Set is not implemented"); - // meta_lbm_info<sch::Prec,sch::Descriptor<3u,19u>>(args_str); - break; - case 19u: - return kel::lbm::make_error<kel::lbm::err::not_implemented>("D3Q17 Velocity Set is not implemented"); - // meta_lbm_info<sch::Prec,sch::Descriptor<3u,19u>>(args_str); - break; - case 27u: - meta_lbm_info<sch::Prec,sch::Descriptor<3u,27u>>(args_str); - break; - default: - return kel::lbm::make_error<kel::lbm::err::critical>("Second positional needs to be a valid momentum set"); - } - } - break; - default: - return kel::lbm::make_error<kel::lbm::err::critical>("First positional needs to be Dimension 1,2 or 3"); - } - - return kel::lbm::make_void(); -} - -int main(int argc, char** argv){ - auto eov = meta_main(argc,argv); - if(eov.is_error()){ - auto& err = eov.get_error(); - - std::cerr<<"[Error]: "<<err.get_category(); - - auto err_msg = err.get_message(); - if(not err_msg.empty()){ - std::cerr<<" - "<<err_msg; - } - - std::cerr<<std::endl; - } - - return 0; -} diff --git a/examples/particles_gpu/.nix/derivation.nix b/examples/particles_gpu/.nix/derivation.nix deleted file mode 100644 index 6ab8825..0000000 --- a/examples/particles_gpu/.nix/derivation.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ lib -, stdenv -, scons -, clang-tools -, forstio -, python3 -, pname -, version -, kel -, adaptive-cpp -}: - -stdenv.mkDerivation { - pname = pname + "-examples-" + "particles"; - inherit version; - src = ./..; - - nativeBuildInputs = [ - scons - clang-tools - python3 - ]; - - buildInputs = [ - forstio.core - forstio.async - forstio.codec - forstio.codec-unit - forstio.codec-json - kel.lbm.core - adaptive-cpp - ]; - - preferLocalBuild = true; - - outputs = [ "out" "dev" ]; -} diff --git a/examples/particles_gpu/SConscript b/examples/particles_gpu/SConscript deleted file mode 100644 index f22fdf2..0000000 --- a/examples/particles_gpu/SConscript +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/false - -import os -import os.path -import glob - - -Import('env') - -dir_path = Dir('.').abspath - -# Environment for base library -examples_env = env.Clone(); -examples_env['CXX'] = 'syclcc-clang'; -examples_env['CXXFLAGS'] += ['-O3']; - -examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) -examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) - -env.sources += examples_env.sources; -env.headers += examples_env.headers; - -# Cavity2D -examples_objects = []; -examples_env.add_source_files(examples_objects, ['particles_gpu.cpp'], shared=False); -examples_env.particles_gpu = examples_env.Program('#bin/particles_gpu', [examples_objects]); - -# Set Alias -env.examples = [ - examples_env.particles_gpu -]; -env.Alias('examples', env.examples); -env.targets += ['examples']; -env.Install('$prefix/bin/', env.examples); diff --git a/examples/particles_gpu/SConstruct b/examples/particles_gpu/SConstruct deleted file mode 100644 index fe206e1..0000000 --- a/examples/particles_gpu/SConstruct +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import os.path -import glob -import re - - -if sys.version_info < (3,): - def isbasestring(s): - return isinstance(s,basestring) -else: - def isbasestring(s): - return isinstance(s, (str,bytes)) - -def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): - - if isbasestring(filetype): - dir_path = self.Dir('.').abspath - filetype = sorted(glob.glob(dir_path+"/"+filetype)) - - for path in filetype: - target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) - if shared: - target_name+='.os' - sources.append( self.SharedObject( target=target_name, source=path ) ) - else: - target_name+='.o' - sources.append( self.StaticObject( target=target_name, source=path ) ) - pass - -def isAbsolutePath(key, dirname, env): - assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) - -env_vars = Variables( - args=ARGUMENTS -) - -env_vars.Add('prefix', - help='Installation target location of build results and headers', - default='/usr/local/', - validator=isAbsolutePath -) - -env_vars.Add('build_examples', - help='If examples should be built', - default="true" -) - -env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], - CPPDEFINES=['SAW_UNIX'], - CXXFLAGS=[ - '-std=c++20', - '-g', - '-Wall', - '-Wextra', - '-isystem', 'AdaptiveCpp' - ], - LIBS=[ - 'forstio-core', - 'acpp-rt', - 'omp' - ] -); -env.__class__.add_source_files = add_kel_source_files -env.Tool('compilation_db'); -env.cdb = env.CompilationDatabase('compile_commands.json'); - -env.objects = []; -env.sources = []; -env.headers = []; -env.targets = []; - -Export('env') -SConscript('SConscript') - -env.Alias('cdb', env.cdb); -env.Alias('all', [env.targets]); -env.Default('all'); - -env.Alias('install', '$prefix') diff --git a/examples/particles_gpu/particles_gpu.cpp b/examples/particles_gpu/particles_gpu.cpp deleted file mode 100644 index 94ec5f1..0000000 --- a/examples/particles_gpu/particles_gpu.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include <kel/lbm/lbm.hpp> -#include <AdaptiveCpp/sycl/sycl.hpp> - - -#include <kel/lbm/particle/particle.hpp> -#include <iostream> - -namespace kel{ -namespace lbm { -namespace sch { -} -} - -kel::lbm::error_or<void> lbm_main(int argc, char** argv){ - (void) argc; - (void) argv; - - using namespace lbm; - using namespace acpp; - - kel::lbm::data<sch::Array<sch::Particle<sch::Float32,2u>>> particles{1024u}; - - for(kel::lbm::data<sch::UInt64> i{0u}; i < kel::lbm::data<sch::UInt64>{32u}; ++i){ - for(kel::lbm::data<sch::UInt64> j{0u}; j < kel::lbm::data<sch::UInt64>{32u}; ++j){ - - auto& part = particles.at(i*32ul+j); - auto& body = part.template get<"rigid_body">(); - - auto& pos = body.template get<"position">(); - auto& old_pos = body.template get<"position_old">(); - auto& acceleration = body.template get<"acceleration">(); - auto& p_rad = part.template get<"collision">().template get<"radius">(); - // Scalar<T> base, so the value goes through the element rather than a - // braced whole-unit assignment. - p_rad.at({}).set(0.4f); - - if(j.get() % 2u == 0) acceleration.at({{1u}}) = {-9.81}; - - pos.at({{0u}}) = {i.template cast_to<sch::Float32>()+0.5f}; - pos.at({{1u}}) = {j.template cast_to<sch::Float32>()+64.0f}; - - old_pos = pos; - } - } - - // A timestep is a lattice time, which makes (pos - pos_old)/time_step a - // velocity rather than an unlabelled vector. - kel::lbm::data<sch::LbmSecond<sch::Scalar<sch::Float32>>> time_step; - time_step.at({}).set(0.05f); - - for(kel::lbm::data<sch::UInt64> dt{0u}; dt < kel::lbm::data<sch::UInt64>{32ul}; ++dt){ - // Do Verlet Step - for(kel::lbm::data<sch::UInt64> i{0u}; i < particles.size(); ++i){ - auto& part_i = particles.at(i); - - verlet_step_lambda<sch::Float32,2u>(part_i,time_step); - auto& body_i = part_i.template get<"rigid_body">(); - auto& acc_i = body_i.template get<"acceleration">(); - - acc_i.at({{0u}}) = 0.0; - acc_i.at({{1u}}) = -9.81; - } - - // - for(kel::lbm::data<sch::UInt64> i{0u}; i < particles.size(); ++i){ - auto& part_i = particles.at(i); - auto& body_i = part_i.template get<"rigid_body">(); - auto& pos_i = body_i.template get<"position">(); - auto& pos_old_i = body_i.template get<"position_old">(); - /** - * Test against other particles - */ - for(kel::lbm::data<sch::UInt64> j{i+1ul}; j < particles.size(); ++j){ - auto& part_j = particles.at(j); - auto& body_j = part_j.template get<"rigid_body">(); - auto& pos_j = body_j.template get<"position">(); - auto& pos_old_j = body_j.template get<"position_old">(); - - auto res = broadphase_collision_distance_squared<sch::Float32,2u>(part_i, part_j); - if(res.first){ - //std::cout<<"Collision"<<std::endl; - // Do collision - - auto vel_i = (pos_i - pos_old_i)/time_step; - auto vel_j = (pos_j - pos_old_j)/time_step; - - auto vel_rel = vel_i - vel_j; - auto pos_rel = pos_i - pos_j; - - auto vel_pos_rel_dot = kel::lbm::math::dot(vel_rel,pos_rel); - if(vel_pos_rel_dot.at({}).get() < 0.0){ - auto pos_rel_normed = pos_rel / kel::lbm::math::sqrt(res.second); - } - - // auto vel_ij = kel::lbm::math::dot(vel_i,vel_j); - - } - } - /** - * Test against walls - */ - if(pos_i.at({{0u}}).get() <= 0 or pos_i.at({{0u}}).get() >= 40 ){ - auto pos_i_0 = pos_i.at({{0u}}); - pos_i.at({{0u}}) = pos_old_i.at({{0u}}); - pos_old_i.at({{0u}}) = pos_i_0; - } - if(pos_i.at({{1u}}).get() <= 0 ){ - auto pos_i_1 = pos_i.at({{1u}}); - pos_i.at({{1u}}) = pos_old_i.at({{1u}}); - pos_old_i.at({{1u}}) = pos_i_1; - } - } - auto& pos = particles.at({0u}).template get<"rigid_body">().template get<"position">(); - - std::cout<<pos.at({{0u}}).get()<<" "<<pos.at({{1u}}).get()<<std::endl; - } - - return kel::lbm::make_void(); -} -} - -int main(int argc, char** argv){ - auto eov = kel::lbm_main(argc, argv); - if(eov.is_error()){ - auto& err = eov.get_error(); - std::cerr<<"[Error] "<<err.get_category(); - auto err_msg = err.get_message(); - if(err_msg.size() > 0u){ - std::cerr<<" - "<<err_msg; - } - std::cerr<<std::endl; - return err.get_id(); - } - - return 0; -} diff --git a/examples/poiseulle_2d_gpu/.nix/derivation.nix b/examples/poiseulle_2d_gpu/.nix/derivation.nix index cc132ee..56f5cae 100644 --- a/examples/poiseulle_2d_gpu/.nix/derivation.nix +++ b/examples/poiseulle_2d_gpu/.nix/derivation.nix @@ -27,10 +27,12 @@ stdenv.mkDerivation { forstio.codec forstio.codec-unit forstio.remote + forstio.remote-filesystem forstio.codec-json forstio.io adaptive-cpp kel.lbm.core + kel.lbm.sycl # kel-lbm.sycl ]; diff --git a/examples/poiseulle_2d_gpu/SConstruct b/examples/poiseulle_2d_gpu/SConstruct index a7201cb..0611b67 100644 --- a/examples/poiseulle_2d_gpu/SConstruct +++ b/examples/poiseulle_2d_gpu/SConstruct @@ -57,7 +57,9 @@ env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], '-Wextra' ], LIBS=[ - 'forstio-core' + 'forstio-core', + 'forstio-async', + 'forstio-io' ] ); env.__class__.add_source_files = add_kel_source_files diff --git a/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp b/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp index 5959506..7e4d901 100644 --- a/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp +++ b/examples/poiseulle_2d_gpu/poiseulle_2d_gpu.cpp @@ -1,395 +1,439 @@ +#include <kel/lbm/sycl/lbm.hpp> #include <kel/lbm/lbm.hpp> -#include <AdaptiveCpp/sycl/sycl.hpp> -#include <forstio/codec/data.hpp> - -template<typename T> -using SyclHostAlloc = acpp::sycl::usm_allocator<kel::lbm::data<T>, acpp::sycl::usm::alloc::host>; +#include <forstio/io/io.hpp> +#include <forstio/remote/filesystem/easy.hpp> +#include <forstio/codec/json/json.hpp> +#include <forstio/codec/simple.hpp> +#include <forstio/codec/args.hpp> namespace kel { namespace lbm { -namespace sch { - -/** - * Basic distribution function - * Base type - * D - * Q - * Scalar factor - * D factor - * Q factor - */ -using T = Float64; -using D2Q9 = Descriptor<2u,9u>; - -using DfCell = Cell<T, D2Q9, 0u, 0u, 1u>; -using CellInfo = Cell<UInt8, D2Q9, 1u, 0u, 0u>; -/** - * Basic type for simulation - */ -using CellStruct = Struct< - Member<DfCell, "dfs">, - Member<DfCell, "dfs_old">, - Member<CellInfo, "info"> ->; +// The channel this case has always used. +constexpr uint64_t dim_x = 1024ul; +constexpr uint64_t dim_y = 512ul; -using MacroStruct = Struct< - Member<FixedArray<Float64,D2Q9::D>, "velocity">, - Member<Float64, "pressure"> ->; +namespace sch { -} +using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y>; -namespace cmpt { -template<bool East> -struct PressureBoundaryRestrictedVelocityTo {}; -} +template<typename T, typename Desc> +using DfChunk = Chunk<FixedArray<T,Desc::Q>, 1u, dim_x, dim_y>; -template<typename FP,typename Desc, bool East> -struct component<FP,Desc, cmpt::PressureBoundaryRestrictedVelocityTo<East>> { -private: - kel::lbm::data<FP> pressure_setting_; - kel::lbm::data<FP> rho_setting_; -public: - component(const kel::lbm::data<FP>& pressure_setting__): - pressure_setting_{pressure_setting__}, - rho_setting_{pressure_setting__ * df_info<FP,Desc>::inv_cs2} - {} +template<typename T, typename Desc> +using ScalarChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; - template<typename CellFieldSchema> - void apply( - kel::lbm::data<CellFieldSchema>* field, - kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index, - const kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>>& meta, - uint64_t time_step - ) const { +template<typename T, typename Desc> +using VectorChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; - using dfi = df_info<FP,Desc>; +template<typename T, typename Desc> +using ChunkStruct = Struct< + Member<InfoChunk, "info">, + Member<DfChunk<T,Desc>, "dfs">, + Member<DfChunk<T,Desc>, "dfs_old"> +>; - auto flat_ind = flatten_index<sch::UInt64,Desc::D>::apply(index,meta); +template<typename T, typename Desc> +using VelChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y>; - bool is_even = ((time_step % 2u) == 0u); - auto& cell = field[flat_ind.get()]; +template<typename T> +using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y>; - auto& info = cell.template get<"info">(); - if(info({0u}).get() == 0u){ - return; - } - auto& dfs_old = (is_even) ? cell.template get<"dfs_old">() : cell.template get<"dfs">(); +template<typename T, typename Desc> +using MacroStruct = Struct< + Member<VelChunk<T,Desc>, "velocity">, + Member<RhoChunk<T>, "density"> +>; - /** - * Sum all known DFs - */ - kel::lbm::data<FP> sum_df{0u}; - for(kel::lbm::data<sch::UInt64> k{0u}; k < kel::lbm::data<sch::UInt64>{Desc::Q}; ++k){ - auto c_k = dfi::directions[k.get()]; - auto flat_ind_n = flatten_index<sch::UInt64,Desc::D>::apply({{index.at({0u})+c_k[0u], index.at({1u})+c_k[1u]}},meta); - auto& cell_n = field[flat_ind_n.get()]; - auto& info_n = cell_n.template get<"info">(); - auto info_n_val = info_n({0u}); - auto k_opp = dfi::opposite_index[k.get()]; +//template<typename T, typename Desc> +//using ParticleArray = Array< +// Particle<T,Desc::D> +//>; +} - if(info_n_val.get() > 0u){ - sum_df += dfs_old({k_opp}); - } - } - /** - * Get the sum of the unknown dfs and precalculate the direction - */ - constexpr int known_dir = East ? 1 : -1; - auto sum_unknown_dfs = (rho_setting_ - sum_df) * kel::lbm::data<FP>{known_dir}; +template<typename T, typename Desc> +kel::lbm::error_or<void> setup_initial_conditions( + kel::lbm::data<sch::ChunkStruct<T,Desc>>& fields, + kel::lbm::data<sch::MacroStruct<T,Desc>>& macros +){ + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u}} + ); + + // Inflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u}}, + {{1u,dim_y}}, + {{0u,1u}} + ); + + // Outflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u}}, + {{dim_x, dim_y}}, + {{0u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& vel_f = macros.template get<"velocity">(); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& vel = vel_f.at(index); + auto eq = equilibrium<T,Desc>(rho,vel); - for(kel::lbm::data<sch::UInt64> k{0u}; k < kel::lbm::data<sch::UInt64>{Desc::Q}; ++k){ - auto c_k = dfi::directions[k.get()]; - auto flat_ind_n = flatten_index<sch::UInt64,Desc::D>::apply({{index.at({0u})+c_k[0u], index.at({1u})+c_k[1u]}},meta); - auto& cell_n = field[flat_ind_n.get()]; - auto& info_n = cell_n.template get<"info">(); - auto info_n_val = info_n({0u}); + df = eq; + }, + {},// 0-index + df_f.get_dims() + ); - if(info_n_val.get() > 0u){ - sum_unknown_dfs += dfs_old({k}) * c_k[0u]; + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& vel = vel_f.at(index); + if(info_f.at(index).get() == 2u){ + vel.at({{0u}}) = 0.0; } - } + auto eq = equilibrium<T,Desc>(rho,vel); - auto vel_x = sum_unknown_dfs / rho_setting_; - - if constexpr (East) { - dfs_old({2u}) = dfs_old({1u}) + kel::lbm::data<FP>{2.0 / 3.0} * rho_setting_ * vel_x; - dfs_old({6u}) = dfs_old({5u}) + kel::lbm::data<FP>{1.0 / 6.0} * rho_setting_ * vel_x + kel::lbm::data<FP>{0.5} * (dfs_old({4u}) - dfs_old({3u})); - dfs_old({8u}) = dfs_old({7u}) + kel::lbm::data<FP>{1.0 / 6.0} * rho_setting_ * vel_x + kel::lbm::data<FP>{0.5} * (dfs_old({3u}) - dfs_old({4u})); - }else if constexpr (not East){ - dfs_old({1u}) = dfs_old({2u}) - kel::lbm::data<FP>{2.0 / 3.0} * rho_setting_ * vel_x; - dfs_old({5u}) = dfs_old({6u}) - kel::lbm::data<FP>{1.0 / 6.0} * rho_setting_ * vel_x + kel::lbm::data<FP>{0.5} * (dfs_old({3u}) - dfs_old({4u})); - dfs_old({7u}) = dfs_old({8u}) - kel::lbm::data<FP>{1.0 / 6.0} * rho_setting_ * vel_x + kel::lbm::data<FP>{0.5} * (dfs_old({4u}) - dfs_old({3u})); - } - } -}; - -/** - * This is massively hacky and expects a lot of conditions - * Either this or mirrored along the horizontal line works - * - * 0 - 2 - 2 - * 0 - 3 - 1 - * 0 - 3 - 1 - * ......... - * 0 - 3 - 1 - * 0 - 2 - 2 - * - */ - -template<typename Desc> -kel::lbm::error_or<void> set_geometry( - kel::lbm::data<sch::CellStruct>* cells, - const kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>>& meta, - acpp::sycl::queue& sycl_q -){ - using namespace kel::lbm; + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u}} + ); - kel::lbm::data<sch::T> rho{1.0}; - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>> vel{{0.0,0.0}}; - auto eq = equilibrium<sch::T,Desc>(rho, vel); - - sycl_q.submit([&](acpp::sycl::handler& h){ - h.parallel_for(acpp::sycl::range<2>{meta.at({0}).get(), meta.at({1}).get()},[=](acpp::sycl::id<2> idx){ - size_t i = idx[0]; - size_t j = idx[1]; - size_t acc_id = j * meta.at({0u}).get() + i; + iterator<Desc::D>::apply( + [&](auto& index){ + kel::lbm::data<sch::Vector<T,Desc::D>> middle, ind_vec; + middle.at({{0u}}) = dim_x * 0.25; + middle.at({{1u}}) = dim_y * 0.5; - auto& c = cells[acc_id]; - auto& info = c.template get<"info">()({0}); - auto& dfs = c.template get<"dfs">(); - auto& dfs_old = c.template get<"dfs_old">(); + ind_vec.at({{0u}}) = index.at({{0u}}).template cast_to<T>(); + ind_vec.at({{1u}}) = index.at({{1u}}).template cast_to<T>(); - if(i >= 2u and j >= 2u and (i+2u) < meta.at({0u}).get() and (j+2u) < meta.at({1u}).get()){ - // Fluid - info.set({2u}); - }else if(((j+2u) == meta.at({1u}).get() or j == 1u) and (i>=1u and (i+1u)<meta.at({0u}).get() )){ - // Wall - info.set({1u}); - }else if((i==1u) and (j >= 1 and (j+1 < meta.at({1u}).get()) ) ){ - // Left input - info.set({3u}); - }else if((i+2u) == meta.at({0u}).get() and (j >= 1 and (j+1) < meta.at({1u}).get() )){ - // Right output - info.set({4u}); - }else { - info.set({0u}); - } - for(kel::lbm::data<sch::UInt64> k{0u}; k < kel::lbm::data<sch::UInt64>{Desc::Q}; ++k){ - dfs(k) = eq.at(k); - dfs_old(k) = eq.at(k); + auto dist = middle - ind_vec; + auto dist_2 = kel::lbm::math::dot(dist,dist); + if(dist_2.at({}).get() < dim_y*dim_y*0.01){ + info_f.at(index) = 1u; } - }); - }).wait(); + }, + {},// 0-index + df_f.get_dims() + ); return kel::lbm::make_void(); } -template<typename Desc> -void step( - kel::lbm::data<sch::CellStruct>* cells, - kel::lbm::data<sch::MacroStruct>* macro_cells, - const kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>>& meta, - uint64_t time_step, - acpp::sycl::queue& sycl_q +template<typename T, typename Desc> +kel::lbm::error_or<void> step( + kel::lbm::data<sch::Ptr<sch::ChunkStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& fields, + kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, + kel::lbm::data<sch::UInt64> t_i, + device& dev ){ - using namespace kel::lbm; - using dfi = df_info<sch::T,Desc>; - - constexpr kel::lbm::data<sch::T> frequency{1.0 / 0.6}; + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); - bool is_even = ((time_step % 2u) == 0u); - /** - * 1. Relaxation parameter \tau - */ - /* - component<sch::T, sch::D2Q9, cmpt::BGK> coll{0.5384}; - component<sch::T, sch::D2Q9, cmpt::BounceBack> bb; - */ - component<sch::T, Desc, cmpt::PressureBoundaryRestrictedVelocityTo<true>> inlet{1.01 * dfi::cs2 * 2.0/3.0}; - component<sch::T, Desc, cmpt::PressureBoundaryRestrictedVelocityTo<false>> outlet{1.0 * dfi::cs2 * 2.0/3.0}; + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + // Need nicer things to handle the flow. I see improvement here + component<T,Desc,cmpt::BGK, encode::Sycl<kel::lbm::enc::Native>> collision{0.8}; + component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; + component<T,Desc,cmpt::AntiBounceBack<0u>,encode::Sycl<kel::lbm::enc::Native>> abb; - - // auto collision_ev = - sycl_q.submit([&](acpp::sycl::handler& h){ - - /// Collision - h.parallel_for(acpp::sycl::range<2>{meta.at({0}).get(), meta.at({1}).get()},[=](acpp::sycl::id<Desc::D> idx){ - size_t i = idx[0]; - size_t j = idx[1]; - size_t acc_id = j * meta.at({0u}).get() + i; - - auto& c = cells[acc_id]; - auto& info = cells[acc_id].template get<"info">(); + kel::lbm::data<sch::Scalar<T>> rho_b; + rho_b.at({}) = 1.0; + kel::lbm::data<sch::Vector<T,Desc::D>> vel_b; + vel_b.at({{0u}}) = 0.015; - switch (info({0u}).get()) { - // Bounce Back - case 1u: { - auto& dfs_old = is_even ? c.template get<"dfs_old">() : c.template get<"dfs">(); - auto df_cpy = dfs_old.copy(); + component<T,Desc,cmpt::Equilibrium,encode::Sycl<kel::lbm::enc::Native>> equi{rho_b,vel_b}; - for(uint64_t k = 1u; k < Desc::Q; ++k){ - dfs_old({k}) = df_cpy({dfi::opposite_index.at(k)}); + component<T,Desc,cmpt::ZouHeHorizontal<true>,encode::Sycl<kel::lbm::enc::Native>> flow_in{ + [&](){ + uint64_t target_t_i = 32u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.0002 / target_t_i) * t_i.get(); } + return 1.0002; + }() + }; + component<T,Desc,cmpt::ZouHeHorizontal<false>,encode::Sycl<kel::lbm::enc::Native>> flow_out{1.0}; - break; - } - // Collision - case 2u: { - // coll.apply(latt_acc, {i, j}, time_step); - auto& dfs_old = is_even ? c.template get<"dfs_old">() : c.template get<"dfs">(); - - auto& macro_c = macro_cells[acc_id]; - - kel::lbm::data<sch::T>& rho = macro_c.template get<"pressure">(); - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"velocity">(); - compute_rho_u<sch::T,Desc>(dfs_old,rho,vel); - auto eq = equilibrium<sch::T,Desc>(rho,vel); - - for(uint64_t k = 0u; k < Desc::Q; ++k){ - dfs_old({k}) = dfs_old({k}) + frequency * (eq.at({k}) - dfs_old({k})); - } - break; + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); } - case 3u: { - inlet.apply(cells, {{i,j}}, meta, time_step); - auto& dfs_old = is_even ? c.template get<"dfs_old">() : c.template get<"dfs">(); - - auto& macro_c = macro_cells[acc_id]; - kel::lbm::data<sch::T>& rho = macro_c.template get<"pressure">(); - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"velocity">(); - - compute_rho_u<sch::T,Desc>(dfs_old,rho,vel); - auto eq = equilibrium<sch::T,Desc>(rho,vel); - - for(uint64_t k = 0u; k < Desc::Q; ++k){ - dfs_old({k}) = dfs_old({k}) + frequency * (eq.at({k}) - dfs_old({k})); - } + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: break; - } - case 4u: { - outlet.apply(cells, {{i,j}}, meta, time_step); - auto& dfs_old = is_even ? c.template get<"dfs_old">() : c.template get<"dfs">(); - - auto& macro_c = macro_cells[acc_id]; - - kel::lbm::data<sch::T>& rho = macro_c.template get<"pressure">(); - kel::lbm::data<sch::FixedArray<sch::T,Desc::D>>& vel = macro_c.template get<"velocity">(); - - compute_rho_u<sch::T,Desc>(dfs_old,rho,vel); - auto eq = equilibrium<sch::T,Desc>(rho,vel); - - for(uint64_t k = 0u; k < Desc::Q; ++k){ - dfs_old({k}) = dfs_old({k}) + frequency * (eq.at({k}) - dfs_old({k})); - } + case 1u: + bb.apply(fields,index,t_i); break; - } - default: - // Do nothing + case 2u: + collision.apply(fields,macros,index,t_i); + break; + case 3u: + flow_in.apply(fields,index,t_i); + //equi.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: break; } }); }).wait(); - - //auto stream_ev = - sycl_q.submit([&](acpp::sycl::handler& h){ - /// Stream - h.parallel_for(acpp::sycl::range<2>{meta.at({0}).get(), meta.at({1}).get()},[=](acpp::sycl::id<Desc::D> idx){ - size_t i = idx[0]; - size_t j = idx[1]; - size_t acc_id = j * meta.at({0u}).get() + i; - - auto& c = cells[acc_id]; - auto& info = c.template get<"info">(); - auto& dfs_new = is_even ? c.template get<"dfs">() : c.template get<"dfs_old">(); - if (info({0u}).get() > 0u) { - for (uint64_t k = 0u; k < Desc::Q; ++k) { - auto dir = dfi::directions[dfi::opposite_index[k]]; - size_t acc_old_id = (j+dir[1]) * meta.at({0u}).get() + (i+dir[0]); - - auto& dfs_old = is_even ? cells[acc_old_id].template get<"dfs_old">() : cells[acc_old_id].template get<"dfs">(); - auto& info_old = cells[acc_old_id].template get<"info">(); - - dfs_new({k}) = dfs_old({k}); - } - } - - }); + + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); }).wait(); + */ + + return kel::lbm::make_void(); } } } template<typename T, typename Desc> -kel::lbm::error_or<void> kel_main(int argc, char** argv){ - using namespace kel; +kel::lbm::error_or<void> lbm_main(int argc, char** argv){ + using namespace kel::lbm; + + /* + auto eo_lbm_args = setup_lbm_env<sch::Args<sch::Struct<>,sch::Tuple<>>>(argc,argv); + if(eo_lbm_args.is_error()){ + return std::move(eo_lbm_args.get_error()); + } + auto& lbm_args = eo_lbm_args.get_value(); + */ - using dfi = lbm::df_info<T,Desc>; + using dfi = df_info<T,Desc>; - auto eo_lbm_dir = lbm::output_directory(); + auto eo_lbm_dir = output_directory(); if(eo_lbm_dir.is_error()){ return std::move(eo_lbm_dir.get_error()); } auto& lbm_dir = eo_lbm_dir.get_value(); - auto out_dir = lbm_dir / "poiseulle_channel_2d_gpu"; - std::filesystem::create_directories(out_dir); - // - lbm::converter<lbm::sch::Float64> conv { + + auto out_dir = lbm_dir / "poiseulle_2d_gpu"; + + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not create output directory"); + } + } + + converter<T> conv { // delta_x {1.0}, // delta_t {1.0} }; + + print_lbm_meta<T,Desc>(conv,{0.1},{1e-4},{0.4 * dim_y}); - uint64_t x_d = 1024u; - uint64_t y_d = 512u; + // kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> meta{{dim_x,dim_y}}; + auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); + auto lbm_macro_data_ptr = kel::lbm::heap<kel::lbm::data<sch::MacroStruct<T,Desc>>>(); + + std::cout<<"Estimated Bytes: "<<memory_estimate<sch::ChunkStruct<T,Desc>,sch::MacroStruct<T,Desc>>().get()<<std::endl; - kel::lbm::data<lbm::sch::FixedArray<lbm::sch::UInt64,Desc::D>> meta{{x_d,y_d}}; + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error<kel::lbm::err::critical>( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; - acpp::sycl::queue sycl_q; - SyclHostAlloc<lbm::sch::MacroStruct> sycl_host_alloc{sycl_q}; - // SyclDeviceAlloc<lbm::sch::CellStruct> sycl_dev_alloc{sycl_q}; + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); - std::vector<kel::lbm::data<lbm::sch::MacroStruct>, SyclHostAlloc<lbm::sch::MacroStruct>> host_cells{x_d * y_d,sycl_host_alloc}; + device dev; - kel::lbm::data<lbm::sch::CellStruct>* cells = acpp::sycl::malloc_device<kel::lbm::data<lbm::sch::CellStruct>>(x_d * y_d,sycl_q); - kel::lbm::data<lbm::sch::MacroStruct>* macro_cells = acpp::sycl::malloc_device<kel::lbm::data<lbm::sch::MacroStruct>>(x_d * y_d,sycl_q); + auto& sycl_q = dev.get_handle(); + + sycl_q.wait(); { - auto eov = lbm::set_geometry<Desc>(cells,meta,sycl_q); + auto eov = setup_initial_conditions<T,Desc>(*lbm_data_ptr,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"initial_state",0u,*lbm_data_ptr); if(eov.is_error()){ return eov; } } + kel::lbm::data<sch::ChunkStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_data{sycl_q}; + kel::lbm::data<sch::MacroStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_macro_data{sycl_q}; + sycl_q.wait(); - uint64_t time_max = 1024u*128u; - for(uint64_t i = 0u; i < time_max; ++i){ - lbm::step<Desc>(cells,macro_cells,meta,i,sycl_q); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + kel::lbm::data<sch::UInt64> time_steps{16u*4096ul}; + + auto& info_f = lsd_view.template get<"info">(); + + for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ + // BC + Collision + { + auto eov = step<T,Desc>(lsd_view,lsdm_view,i,dev); + if(eov.is_error()){ + return eov; + } + } sycl_q.wait(); - if(i%4u == 0u){ - std::string vtk_f_name{"tmp/poiseulle_2d_gpu_"}; - vtk_f_name += std::to_string(i) + ".vtk"; - // write_vtk_file(vtk_f_name,host_cells); - sycl_q.memcpy(&host_cells[0u], macro_cells, x_d * y_d * sizeof(kel::lbm::data<lbm::sch::MacroStruct>) ).wait(); - lbm::write_vtk_file<lbm::sch::MacroStruct,Desc::D>(vtk_f_name, &host_cells[0], meta); + /* + if(i.get() % 128u == 0u){ + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + */ + /* + if(i.get() % 32u == 0u){ + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_csv_file(out_dir,"m",i.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + } + */ + // Stream + sycl_q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::Stream,encode::Sycl<kel::lbm::enc::Native>> stream; + + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(lsd_view,index,i); + } + }); + }).wait(); + wait.poll(); + if(print_status){ + std::cout<<"Status: "<<i.get()<<" of "<<time_steps.get()<<" - "<<(i.template cast_to<sch::Float64>().get() * 100 / time_steps.get())<<"%"<<std::endl; + print_status = false; } + print_progress_bar(i.get(), time_steps.get()-1u); } + // After Loop sycl_q.wait(); - acpp::sycl::free(cells, sycl_q); - acpp::sycl::free(macro_cells, sycl_q); - sycl_q.wait(); + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_vtk_file(out_dir,"m",time_steps.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = write_csv_file(out_dir,"m",time_steps.get(), *lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); return kel::lbm::make_void(); } +using FloatT = kel::lbm::sch::Float32; + int main(int argc, char** argv){ - auto eov = kel_main<kel::lbm::sch::T,kel::lbm::sch::D2Q9>(argc, argv); + auto eov = lbm_main<FloatT,kel::lbm::sch::D2Q9>(argc, argv); if(eov.is_error()){ auto& err = eov.get_error(); std::cerr<<"[Error] "<<err.get_category(); diff --git a/examples/poiseulle_3d/.nix/derivation.nix b/examples/poiseulle_3d/.nix/derivation.nix index b090add..02fcb8c 100644 --- a/examples/poiseulle_3d/.nix/derivation.nix +++ b/examples/poiseulle_3d/.nix/derivation.nix @@ -21,14 +21,16 @@ stdenv.mkDerivation { buildInputs = [ forstio.core + forstio.remote-filesystem + forstio.io forstio.async forstio.codec forstio.codec-unit forstio.codec-json forstio.remote - forstio.remote-sycl adaptive-cpp kel.lbm.core + kel.lbm.sycl ]; preferLocalBuild = true; diff --git a/examples/poiseulle_3d/SConstruct b/examples/poiseulle_3d/SConstruct index fe206e1..7882265 100644 --- a/examples/poiseulle_3d/SConstruct +++ b/examples/poiseulle_3d/SConstruct @@ -59,6 +59,8 @@ env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], ], LIBS=[ 'forstio-core', + 'forstio-async', + 'forstio-io', 'acpp-rt', 'omp' ] diff --git a/examples/poiseulle_3d/poiseulle_3d.cpp b/examples/poiseulle_3d/poiseulle_3d.cpp index 141dc5f..34d1fbc 100644 --- a/examples/poiseulle_3d/poiseulle_3d.cpp +++ b/examples/poiseulle_3d/poiseulle_3d.cpp @@ -1,428 +1,431 @@ +#include <kel/lbm/sycl/lbm.hpp> #include <kel/lbm/lbm.hpp> +#include <kel/lbm/particle.hpp> -#include <forstio/remote/sycl/sycl.hpp> - -namespace saw { -/* -template<typename Desc, typename CellT, typename Encode> -class data<kel::lbm::sch::CellField<Desc, CellT>, encode::Sycl<Encode>> final { -public: - using Schema = kel::lbm::sch::CellField<Desc,CellT>; - using MetaSchema = typename meta_schema<Schema>::MetaSchema; -private: - data<schema::Array<CellT,Desc::D>, encode::Sycl<Encode>> inner_; -public: - data() = delete; - data(const data<MetaSchema,Encode>& inner_meta__, acpp::sycl::queue& q): - inner_{inner_meta__,q} - {} - - const data<MetaSchema, Encode> meta() const { - return inner_.dims(); - } - - template<uint64_t i> - data<schema::UInt64,Encode> get_dim_size() const { - static_assert(i < Desc::D, "Not enough dimensions"); - return inner_.template get_dim_size<i>(); - } - - const data<CellT>& operator()(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index)const{ - return inner_.at(index); - } - - data<CellT>& operator()(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index){ - return inner_.at(index); - } - - const data<CellT>& at(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index)const{ - return inner_.at(index); - } - - data<CellT>& at(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index){ - return inner_.at(index); - } - - ref<data<schema::Array<CellT,Desc::D>, encode::Sycl<Encode> >> inner_data() { - return {inner_}; - } - - data<schema::UInt64,Encode> internal_size() const { - return inner_.internal_size(); - } - - data<CellT,Encode>* internal_data() { - return inner_.internal_data(); - } -}; -*/ -/* -template<typename T, uint64_t D, typename Encode> -class data<schema::Ref<schema::Array<T, D>>, encode::Sycl<Encode>> final { -public: - using Schema = schema::Ref<schema::Array<T, D>>; -private: - data<schema::Ref<schema::Array<CellT,Desc::D>>, encode::Sycl<Encode>>> inner_ref_; -public: - data() = delete; - data(ref<data<kel::lbm::sch::CellField<Desc,CellT>,encode::Sycl<Encode>>> inner_ref__): - inner_ref_{inner_ref__().inner_data()} - {} - - const data<MetaSchema, Encode> meta() const { - return inner_ref_().dims(); - } - - const data<CellT>& operator()(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index)const{ - return inner_ref_().at(index); - } - - data<CellT>& operator()(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index){ - return inner_ref_().at(index); - } - - const data<CellT>& at(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index)const{ - return inner_ref_().at(index); - } - - data<CellT>& at(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index){ - return inner_ref_().at(index); - } - - data<schema::UInt64,Encode> internal_size() const { - return inner_ref_().internal_size(); - } - - data<CellT,Encode>* internal_data() { - return inner_ref_().internal_data(); - } -}; - - -template<typename Desc, typename CellT, typename Encode> -class data<schema::Ref<kel::lbm::sch::CellField<Desc, CellT>>, encode::Sycl<Encode>> final { -public: - using Schema = schema::Ref<kel::lbm::sch::CellField<Desc,CellT>>; -private: - data<schema::Ref<schema::Array<CellT,Desc::D>>, encode::Sycl<Encode>>> inner_ref_; -public: - data() = delete; - data(ref<data<kel::lbm::sch::CellField<Desc,CellT>,encode::Sycl<Encode>>> inner_ref__): - inner_ref_{inner_ref__().inner_data()} - {} - - const data<MetaSchema, Encode> meta() const { - return inner_ref_().dims(); - } - - const data<CellT>& operator()(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index)const{ - return inner_ref_().at(index); - } - - data<CellT>& operator()(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index){ - return inner_ref_().at(index); - } - - const data<CellT>& at(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index)const{ - return inner_ref_().at(index); - } - - data<CellT>& at(const data<schema::FixedArray<schema::UInt64, Desc::D>, Encode>& index){ - return inner_ref_().at(index); - } - - data<schema::UInt64,Encode> internal_size() const { - return inner_ref_().internal_size(); - } - - data<CellT,Encode>* internal_data() { - return inner_ref_().internal_data(); - } -}; -*/ - -} +#include <forstio/io/io.hpp> +#include <forstio/remote/filesystem/easy.hpp> +#include <forstio/codec/json/json.hpp> +#include <forstio/codec/simple.hpp> namespace kel { namespace lbm { -namespace sch { -using T = Float32; -using Desc = Descriptor<3u,27u>; -using DfCell = Cell<T, Desc, 0u, 0u, 1u>; +// The channel this case has always used. +constexpr uint64_t dim_x = 2048ul; +constexpr uint64_t dim_y = 128ul; +constexpr uint64_t dim_z = 128ul; +constexpr uint64_t particle_size = 4ul; -using CellInfo = Cell<UInt8, Desc, 1u, 0u, 0u>; +namespace sch { -using CellStruct = Struct< - Member<DfCell, "dfs">, - Member<DfCell, "dfs_old">, - //Member<CellForceField<Desc>, "force">, - Member<CellInfo, "info"> +using LbmArgs = Args< + Struct< + Member<String, "name">, + Member<UInt64, "time_steps"> + >, + Tuple<> >; -} -using SyclKelAllocator = acpp::sycl::usm_allocator<kel::lbm::data<sch::CellStruct>, acpp::sycl::usm::alloc::shared>; -uint64_t to_flat_index(const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index, const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& meta){ - auto flat = index.at({2u}) + meta.at({2u})*(index.at({1u}) + index.at({0u})*meta.at({1u})); - return flat.get(); -} - -template<typename StructFieldSchema> -kel::lbm::error_or<void> set_geometry(std::vector<kel::lbm::data<StructFieldSchema>,SyclKelAllocator>& latt, const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& meta){ - using namespace kel::lbm; +using InfoChunk = Chunk<UInt8, 0u, dim_x, dim_y, dim_z>; - /** - * Set ghost - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); +template<typename T, typename Desc> +using DfChunk = Chunk<FixedArray<T,Desc::Q>, 1u, dim_x, dim_y, dim_z>; - info({0u}).set(0u); - }, {{0u,0u}}, meta); +template<typename T, typename Desc> +using ScalarChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; - kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>> distance; - - /** - * Set wall - */ - iterator<1u>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,1u>>& index){ - distance.at(index.at({0u})) = {1u}; - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); +template<typename T, typename Desc> +using VectorChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; - info({0u}).set(2u); - }, {}, meta, distance); - - /** - * Set fluid - */ - iterator<1u>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,1u>>& index){ - distance.at(index.at({0u})) = {2u}; - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); +template<typename T, typename Desc> +using ChunkStruct = Struct< + Member<InfoChunk, "info">, + Member<DfChunk<T,Desc>, "dfs">, + Member<DfChunk<T,Desc>, "dfs_old">, + Member<VectorChunk<T,Desc>, "particle_N">, + Member<ScalarChunk<T,Desc>, "particle_D"> +>; - info({0u}).set(1u); +template<typename T, typename Desc> +using MomChunk = Chunk<Vector<T,Desc::D>, 0u, dim_x, dim_y, dim_z>; - }, {}, meta, distance); +template<typename T> +using RhoChunk = Chunk<Scalar<T>, 0u, dim_x, dim_y, dim_z>; - /** - * Set inflow - */ +template<typename T, typename Desc> +using MacroStruct = Struct< + Member<MomChunk<T,Desc>, "velocity">, + Member<RhoChunk<T>, "density">, + Member<ScalarChunk<T,Desc>, "porosity"> +>; +} - kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>> shifted_point; - iterator<1u>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,1u>>& index){ - distance.at(index.at({0u})) = (index.at({0u}).get() == 0u)?1u:2u; - shifted_point.at(index.at({0u})) = (index.at({0u}).get() == 0u)?3u:meta.at({index.at({0u})}); - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); +template<typename T, typename Desc> +kel::lbm::error_or<void> setup_initial_conditions( + converter<T>& conv, + kel::lbm::data<sch::ChunkStruct<T,Desc>>& fields, + kel::lbm::data<sch::MacroStruct<T,Desc>>& macros, + kel::lbm::data<sch::FixedArray<sch::Particle<T,Desc::D>, particle_size>>& particles +){ + // Unit conversion is not exercised in this reduced case. + (void) conv; - info({0u}).set(3u); + auto& info_f = fields.template get<"info">(); + // Set everything as walls + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(1u); + }, + {}, + info_f.get_dims(), + {} + ); + // Fluid + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(2u); + }, + {}, + info_f.get_dims(), + {{1u,1u,1u}} + ); + + // Inflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(3u); + }, + {{0u,0u,0u}}, + {{1u,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + + // Outflow + iterator<Desc::D>::apply( + [&](auto& index){ + info_f.at(index).set(4u); + }, + {{dim_x-1u,0u,0u}}, + {{dim_x,dim_y,dim_z}}, + {{0u,1u,1u}} + ); + // + auto& df_f = fields.template get<"dfs_old">(); + auto& rho_f = macros.template get<"density">(); + auto& mom_f = macros.template get<"velocity">(); + auto& por_f = macros.template get<"porosity">(); + + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + por_f.at(index).at({}) = {1}; + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + auto eq = equilibrium<T,Desc>(rho,mom); - }, {}, shifted_point, distance); + df = eq; + }, + {},// 0-index + df_f.get_dims() + ); - /** - * Set outflow - */ - iterator<1u>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,1u>>& index){ - distance.at(index.at({0u})) = (index.at({0u}).get() == 0u)?1u:2u; - shifted_point.at(index.at({0u})) = (index.at({0u}).get() == 0u)?(meta.at({0u})-3u):0u; - },{{0u}},{{sch::Desc::D}}); - /* - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); + iterator<Desc::D>::apply( + [&](auto& index){ + auto& df = df_f.at(index); + auto& rho = rho_f.at(index); + rho.at({}) = {1}; + auto& mom = mom_f.at(index); + mom.at({{0u}}) = 0.01; + auto eq = equilibrium<T,Desc>(rho,mom); - info({0u}).set(4u); + df = eq; + }, + {},// 0-index + df_f.get_dims(), + {{1u,1u,1u}} + ); - }, shifted_point, meta, distance); + // No particles in this case -- the array is carried only so the schema + // matches poiseulle_3d_gpu. + (void) particles; return kel::lbm::make_void(); } -template<typename StructFieldSchema> -kel::lbm::error_or<void> set_initial_conditions(std::vector<kel::lbm::data<StructFieldSchema>,SyclKelAllocator>& latt, const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& meta){ - - kel::lbm::data<sch::T> rho{1.0}; - kel::lbm::data<sch::FixedArray<sch::T,sch::Desc::D>> vel{}; - vel.at({0u}) = {0.1f}; - auto eq = equilibrium<sch::T,sch::Desc>(rho, vel); - - /** - * Set distribution - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& dfs = cell.template get<"dfs">(); - auto& dfs_old = cell.template get<"dfs_old">(); - - for(kel::lbm::data<sch::UInt64> k = 0; k < kel::lbm::data<sch::UInt64>{sch::Desc::Q}; ++k){ - dfs(k) = eq.at(k); - dfs_old(k) = eq.at(k); - } - }, {}, meta); - return kel::lbm::make_void(); -} - -template<typename StructFieldSchema> -kel::lbm::error_or<void> lbm_step( - std::vector<kel::lbm::data<StructFieldSchema>,SyclKelAllocator>& latt, - const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& meta, - kel::lbm::data<sch::UInt64> time_step, - acpp::sycl::queue& sycl_q +template<typename T, typename Desc> +kel::lbm::error_or<void> step( + converter<T>& conv, + kel::lbm::data<sch::Ptr<sch::ChunkStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& fields, + kel::lbm::data<sch::Ptr<sch::MacroStruct<T,Desc>>,encode::Sycl<kel::lbm::enc::Native>>& macros, + kel::lbm::data<sch::UInt64> t_i, + device& dev ){ - using dfi = df_info<sch::T,sch::Desc>; - bool even_step = ((time_step.get() % 2u) == 0u); + auto& q = dev.get_handle(); + auto& info_f = fields.template get<"info">(); - component<sch::T, sch::Desc, cmpt::BGK, kel::lbm::enc::Native> coll{0.5384}; - component<sch::T, sch::Desc, cmpt::BounceBack, kel::lbm::enc::Native> bb; + // auto coll_ev = + q.submit([&](acpp::sycl::handler& h){ + // Need nicer things to handle the flow. I see improvement here + component<T,Desc,cmpt::BGK, encode::Sycl<kel::lbm::enc::Native>> collision{conv.template kinematic_viscosity_si_to_tau<Desc>({0.1})}; + // component<T,Desc,cmpt::HLBM,encode::Sycl<kel::lbm::enc::Native>> collision{0.6}; + component<T,Desc,cmpt::BounceBack,encode::Sycl<kel::lbm::enc::Native>> bb; - /** - * Collision - */ - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& info = cell.template get<"info">(); + component<T,Desc,cmpt::ZouHeHorizontal<true>,encode::Sycl<kel::lbm::enc::Native>> flow_in{ + [&](){ + uint64_t target_t_i = 256u; + if(t_i.get() < target_t_i){ + return 1.0 + (0.001 / target_t_i) * t_i.get(); + } + return 1.001; + }() + }; + component<T,Desc,cmpt::ZouHeHorizontal<false>,encode::Sycl<kel::lbm::enc::Native>> flow_out{1.0}; - switch(info({0u}).get()){ - case 1u: { - bb.apply(&latt[0u], meta, index, time_step); - break; + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); } - case 2u: { - coll.apply(&latt[0u], meta, index, time_step); + + auto info = info_f.at(index); + + switch(info.get()){ + case 0u: break; - } - case 3u: { - //inlet.apply(latt, index, time_step); + case 1u: + bb.apply(fields,index,t_i); break; - } - case 4u: { - //outlet.apply(latt, index, time_step); + case 2u: + collision.apply(fields,macros,index,t_i); break; - } - default: + case 3u: + flow_in.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); break; - } + case 4u: + flow_out.apply(fields,index,t_i); + collision.apply(fields,macros,index,t_i); + break; + default: + break; + } + }); + }).wait(); - }, {}, meta); + q.submit([&](acpp::sycl::handler& h){ + component<T,Desc,cmpt::Stream,encode::Sycl<kel::lbm::enc::Native>> stream; - // Stream - iterator<sch::Desc::D>::apply([&](const kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>>& index){ - auto findex = to_flat_index(index,meta); - auto& cell = latt.at(findex); - auto& df_new = even_step ? cell.template get<"dfs">() : cell.template get<"dfs_old">(); - auto& info_new = cell.template get<"info">(); + h.parallel_for(acpp::sycl::range<Desc::D>{dim_x,dim_y,dim_z}, [=](acpp::sycl::id<Desc::D> idx){ + kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> index; + for(uint64_t i = 0u; i < Desc::D; ++i){ + index.at({{i}}).set(idx[i]); + } + + auto info = info_f.at(index); + + if(info.get() > 0u){ + stream.apply(fields,index,t_i); + } + }); + }).wait(); - if(info_new({0u}).get() > 0u){ - for(uint64_t k = 0u; k < sch::Desc::Q; ++k){ - auto dir = dfi::directions[dfi::opposite_index[k]]; - auto index_dir = index; - for(uint64_t i = 0; i < sch::Desc::D; ++i){ - index_dir.at({i}) = index_dir.at({i}) + dir[i]; - } + // Step + /* + q.submit([&](acpp::sycl::handler& h){ + // h.depends_on(collision_ev); + }).wait(); + */ - auto findex_dir = to_flat_index(index_dir,meta); - auto& cell_dir_old = latt.at(findex_dir); + return kel::lbm::make_void(); +} +} +} - auto& df_old = even_step ? cell_dir_old.template get<"dfs_old">() : cell_dir_old.template get<"dfs">(); +template<typename T, typename Desc> +kel::lbm::error_or<void> lbm_main(int argc, char** argv){ + using namespace kel::lbm; - df_new({k}) = df_old({k}); - } + kel::lbm::data<sch::LbmArgs> args; + { + auto& args_n = args.template get<"args">(); + auto& name = args_n.template get<"name">(); + name.set("poiseulle_3d"); + auto& steps_arg = args_n.template get<"time_steps">(); + steps_arg.set(64u); + } + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; } - }, {}, meta); - return kel::lbm::make_void(); -} + } + auto eo_lbm_env = init_lbm_env(args); + if(eo_lbm_env.is_error()){ + return std::move(eo_lbm_env.get_error()); + } + const auto& lbm_env = eo_lbm_env.get_value(); + + auto& out_dir = lbm_env.name_dir; -kel::lbm::error_or<void> real_main(int argc, char** argv){ - using dfi = df_info<sch::T,sch::Desc>; + { + std::error_code ec; + std::filesystem::create_directories(out_dir,ec); + if(ec != std::errc{}){ + return kel::lbm::make_error<kel::lbm::err::critical>("Could not create output directory"); + } + } - converter<sch::T> conv { - {0.1f}, - {0.1f} + converter<T> conv { + // delta_x + {1.0}, + // delta_t + {1.0} }; - print_lbm_meta<sch::T,sch::Desc>(conv, {1e-3f}); - acpp::sycl::queue sycl_q; + print_lbm_meta<T,Desc>(conv,{0.1},{0.01},{1024.0}); + + auto lbm_data_ptr = kel::lbm::heap<kel::lbm::data<sch::ChunkStruct<T,Desc>>>(); + auto lbm_macro_data_ptr = kel::lbm::heap<kel::lbm::data<sch::MacroStruct<T,Desc>>>(); + auto lbm_particle_data_ptr = kel::lbm::heap<kel::lbm::data<sch::FixedArray<sch::Particle<T,Desc::D>, particle_size>>>(); + + std::cout<<"Estimated Bytes: "<<memory_estimate<sch::ChunkStruct<T,Desc>,sch::MacroStruct<T,Desc>>().get()<<std::endl; + + auto eo_aio = saw::setup_async_io(); + if(eo_aio.is_error()){ + return kel::lbm::make_error<kel::lbm::err::critical>( + std::string{eo_aio.get_error().get_message()}); + } + auto& aio = eo_aio.get_value(); + saw::wait_scope wait{aio.event_loop}; + + bool krun = true; + bool print_status = false; + aio.event_port.on_signal(saw::Signal::Terminate).then([&](){ + krun = false; + }).detach(); + aio.event_port.on_signal(saw::Signal::User1).then([&](){ + print_status = true; + }).detach(); - /** - * Set the meta and initialize the lattice - */ - kel::lbm::data<sch::FixedArray<sch::UInt64,sch::Desc::D>> meta{{2048u,128u,128u}}; + device dev; - SyclKelAllocator sycl_shared_alloc{sycl_q}; - std::vector<kel::lbm::data<sch::CellStruct>,SyclKelAllocator> lattice{(meta.at({0u})*meta.at({1u})*meta.at({2u})).get(), sycl_shared_alloc}; + auto& sycl_q = dev.get_handle(); - /** - * Set the geometry - */ + sycl_q.wait(); { - auto eov = set_geometry(lattice, meta); + auto eov = setup_initial_conditions<T,Desc>(conv,*lbm_data_ptr,*lbm_macro_data_ptr,*lbm_particle_data_ptr); if(eov.is_error()){ return eov; } } - /** - * Set initial conditions in the simulations - */ + kel::lbm::data<sch::ChunkStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_data{sycl_q}; + kel::lbm::data<sch::MacroStruct<T,Desc>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_macro_data{sycl_q}; + kel::lbm::data<sch::FixedArray<sch::Particle<T,Desc::D>, particle_size>, encode::Sycl<kel::lbm::enc::Native>> lbm_sycl_particle_data{sycl_q}; + sycl_q.wait(); + + auto lsd_view = make_view(lbm_sycl_data); + auto lsdm_view = make_view(lbm_sycl_macro_data); + { + auto eov = dev.copy_to_device(*lbm_data_ptr,lbm_sycl_data); + if(eov.is_error()){ + return eov; + } + } { - auto eov = set_initial_conditions(lattice, meta); + auto eov = dev.copy_to_device(*lbm_macro_data_ptr,lbm_sycl_macro_data); if(eov.is_error()){ return eov; } } + { + auto eov = dev.copy_to_device(*lbm_particle_data_ptr,lbm_sycl_particle_data); + if(eov.is_error()){ + return eov; + } + } + sycl_q.wait(); + kel::lbm::data<sch::UInt64> time_steps = args.template get<"args">().template get<"time_steps">(); - for(kel::lbm::data<sch::UInt64> ts{0u}; ts < kel::lbm::data<sch::UInt64>{1024u*32u}; ++ts){ - auto eov = lbm_step(lattice,meta,ts,sycl_q); + for(kel::lbm::data<sch::UInt64> i{0u}; i < time_steps and krun; ++i){ + { + { + auto eov = write_vtk_file(out_dir,"t",i.get(),*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + /*{ + std::string file_name = "p_"; + file_name += std::to_string(i.get()); + file_name += ".json"; + auto eov = saw::easy::encode_and_write_file<sch::FixedArray<sch::Particle<T,Desc::D>,particle_size>,saw::encode::Json>(out_dir, *lbm_particle_data_ptr); + if(eov.is_error()){ + return eov; + } + }*/ + } + { + auto eov = step<T,Desc>(conv,lsd_view,lsdm_view,i,dev); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = dev.copy_to_host(lbm_sycl_data,*lbm_data_ptr); + if(eov.is_error()){ + return eov; + } + } + + wait.poll(); + if(print_status){ + std::cout<<"Status: "<<i.get()<<" of "<<time_steps.get()<<" - "<<(i.template cast_to<sch::Float64>().get() * 100 / time_steps.get())<<"%"<<std::endl; + print_status = false; + } + } + sycl_q.wait(); + { + auto eov = write_vtk_file(out_dir,"t",time_steps.get(),*lbm_macro_data_ptr); if(eov.is_error()){ return eov; } } + /* + iterator<Desc::D>::apply( + [&](auto& index){ + std::cout<<index.at({0u}).get()<<" "<<index.at({1u}).get()<<" "<<lbm_data.get<"info">().at(index).template cast_to<sch::UInt16>().get()<<"\n"; + }, + {{0u,0u}}, + {{dim_x, dim_y}} + ); + */ + + sycl_q.wait(); return kel::lbm::make_void(); } -} // lbm -} // kel +using FloatT = kel::lbm::sch::Float32; -/** - * main, but I don't like the error handling - */ int main(int argc, char** argv){ - auto eov = kel::lbm::real_main(argc, argv); + auto eov = lbm_main<FloatT,kel::lbm::sch::D3Q27>(argc, argv); if(eov.is_error()){ auto& err = eov.get_error(); + std::cerr<<"[Error] "<<err.get_category(); auto err_msg = err.get_message(); - std::cerr<<"[Error]: "<<err.get_category(); - - if(not err_msg.empty()){ + if(err_msg.size() > 0u){ std::cerr<<" - "<<err_msg; } std::cerr<<std::endl; - return err.get_id(); } - return 0; } diff --git a/examples/stdkr_2d_bgk/sim.cpp b/examples/stdkr_2d_bgk/sim.cpp index b388e3c..d455c55 100644 --- a/examples/stdkr_2d_bgk/sim.cpp +++ b/examples/stdkr_2d_bgk/sim.cpp @@ -2,13 +2,14 @@ #include "init.hpp" #include "step.hpp" -#include <forstio/codec/args.hpp> /** * For deciding what parameters to use maybe? */ namespace args { +using namespace kel::lbm::sch; + using LbmArgsStruct = Struct< >; @@ -193,7 +194,7 @@ kel::lbm::error_or<void> k_main(int argc, char** argv){ kel::lbm::data<args::LbmArgs> args; { - auto eov = saw::parse_args(args,argc,argv); + auto eov = kel::lbm::parse_args(args,argc,argv); if(eov.is_error()){ return eov; } diff --git a/examples/stdkr_2d_bgk/step.hpp b/examples/stdkr_2d_bgk/step.hpp index d7d4fd4..a127bf6 100644 --- a/examples/stdkr_2d_bgk/step.hpp +++ b/examples/stdkr_2d_bgk/step.hpp @@ -47,7 +47,7 @@ kel::lbm::error_or<void> step( [&]() -> kel::lbm::data<sch::Vector<T,Desc::D>> { kel::lbm::data<sch::Vector<T,Desc::D>> vel; { - auto y_si = conv.meter_lbm_to_si(index.at({{1u}}).template cast_to<T>()).handle(); + auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((1.2 / 0.41) * y_si.get() - (1.2 / (0.41*0.41)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } diff --git a/examples/throat_clog_2d/init.hpp b/examples/throat_clog_2d/init.hpp index fab51b3..7683942 100644 --- a/examples/throat_clog_2d/init.hpp +++ b/examples/throat_clog_2d/init.hpp @@ -157,47 +157,23 @@ kel::lbm::error_or<void> init( ); { + // A radius is a lattice length now. kel::lbm::data<sch::LbmMeter<sch::Scalar<T>>> radius_p; radius_p.at({}).set(conv.meter_si_to_lbm({0.05}).handle().get()); std::cout<<"RADIUS: "<<radius_p.at({}).get()<<std::endl; kel::lbm::data<sch::Scalar<T>> dense_p; dense_p.at({}).set(1); - particles = create_spheroid_particle_group<T,Desc::D,particle_amount>(radius_p, dense_p, {{64u}}); + particles = create_spheroid_particle_group<T,Desc::D,particle_amount>(radius_p, dense_p, {64u}); auto& parts = particles.template get<"particles">(); - /** - * Seed the particles upstream of the throat on a staggered grid. - * - * Spacing is 2.4 radii, so they start clear of one another -- any - * initial overlap would be resolved as a contact on the first step and - * fire them apart. - */ - const auto radius = radius_p.at({}).get(); - const auto pitch = radius * 2.4; - - const auto x0 = conv.meter_si_to_lbm({0.10}).handle().get(); - const auto y_mid = conv.meter_si_to_lbm({0.25}).handle().get(); - - constexpr uint64_t per_column = 4ul; - - for(uint64_t i = 0u; i < particle_amount; ++i){ - auto col = i / per_column; - auto row = i % per_column; - - // Offset alternate columns by half a pitch so they interleave. - auto y_off = (static_cast<double>(row) - (per_column - 1u) * 0.5) * pitch - + ((col % 2u == 1u) ? pitch * 0.5 : 0.0); - - auto& p = parts.at({i}); + { + auto& p = parts.at({0u}); auto& prb = p.template get<"rigid_body">(); auto& p_pos = prb.template get<"position">(); auto& p_pos_old = prb.template get<"position_old">(); + p_pos.at({{0u}}) = conv.meter_si_to_lbm({0.1}).handle(); + p_pos.at({{1u}}) = conv.meter_si_to_lbm({0.32}).handle(); - p_pos.at({{0u}}).set(x0 + static_cast<double>(col) * pitch); - p_pos.at({{1u}}).set(y_mid + y_off); - - // Start at rest: position_old defaults to zero, which would read - // as an enormous velocity. p_pos_old = p_pos; } } diff --git a/examples/throat_clog_2d/sim.cpp b/examples/throat_clog_2d/sim.cpp index a25c1a3..04d0092 100644 --- a/examples/throat_clog_2d/sim.cpp +++ b/examples/throat_clog_2d/sim.cpp @@ -2,17 +2,31 @@ #include "init.hpp" #include "step.hpp" -#include <string_view> +#include <forstio/codec/args.hpp> /** - * The coupling method is the only thing this example takes from the command - * line, so it is read straight from argv rather than through a codec -- the - * args codec is forstio's and would need a schema round trip for one string. + * For deciding what parameters to use maybe? */ +namespace args { +using namespace kel::lbm::sch; + +using LbmArgsStruct = Struct< + Member<String, "coupling"> + //,Member<UInt64, "refinement"> +>; + +using LbmArgs = Args< + LbmArgsStruct, + Tuple<> +>; +} + template<typename T, typename Desc, typename Coll, uint64_t N> -kel::lbm::error_or<void> lbm_main(std::string_view coupling){ +kel::lbm::error_or<void> lbm_main(const kel::lbm::data<args::LbmArgs>& args){ using namespace kel::lbm; + auto& an = args.template get<"args">(); + using dfi = df_info<T,Desc>; auto eo_lbm_dir = output_directory(); @@ -21,7 +35,7 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ } auto& lbm_dir = eo_lbm_dir.get_value(); - auto out_dir = lbm_dir / "throat_clog_2d" / coupling; + auto out_dir = lbm_dir / "throat_clog_2d" / an.template get<"coupling">().stl_view(); { std::error_code ec; @@ -49,6 +63,7 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ auto eo_aio = saw::setup_async_io(); if(eo_aio.is_error()){ + // forstio io reports its own error type; translate at the boundary. return kel::lbm::make_error<kel::lbm::err::critical>( std::string{eo_aio.get_error().get_message()}); } @@ -116,7 +131,7 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ auto lsdm_view = make_view(lbm_sycl_macro_data); auto lsdp_view = make_view(lbm_sycl_particle_data); - kel::lbm::data<sch::UInt64> time_steps{256u*512ul}; + kel::lbm::data<sch::UInt64> time_steps{1024ul}; auto& info_f = lsd_view.template get<"info">(); @@ -130,7 +145,7 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ } sycl_q.wait(); - if(i.get() % 256u == 0u){ + if(i.get() % 1u == 0u){ { auto eov = dev.copy_to_host(lbm_sycl_macro_data,*lbm_macro_data_ptr); if(eov.is_error()){ @@ -138,7 +153,7 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ } } { - auto eov = write_vtk_file(out_dir,std::string{"throat_clog_2d_"}+std::string{coupling},i.get(), *lbm_macro_data_ptr); + auto eov = write_vtk_file(out_dir,std::string{"throat_clog_2d_"}+an.template get<"coupling">().stl_string(),i.get(), *lbm_macro_data_ptr); if(eov.is_error()){ return eov; } @@ -178,7 +193,7 @@ kel::lbm::error_or<void> lbm_main(std::string_view coupling){ } } { - auto eov = write_vtk_file(out_dir,std::string{"throat_clog_2d_"}+std::string{coupling},time_steps.get(), *lbm_macro_data_ptr); + auto eov = write_vtk_file(out_dir,std::string{"throat_clog_2d_"}+an.template get<"coupling">().stl_string(),time_steps.get(), *lbm_macro_data_ptr); if(eov.is_error()){ return eov; } @@ -198,36 +213,37 @@ using DescT = kel::lbm::sch::D2Q9; kel::lbm::error_or<void> k_main(int argc, char** argv){ using namespace kel::lbm; - - // Default to fplbm so the example runs with no arguments at all. - std::string_view coupling{"fplbm"}; - if(argc > 1){ - coupling = argv[1]; + + kel::lbm::data<args::LbmArgs> args; + { + auto eov = kel::lbm::parse_args(args,argc,argv); + if(eov.is_error()){ + return eov; + } } + auto& an = args.template get<"args">(); + auto& coupling = an.template get<"coupling">(); auto resolution_resolve = [&]<typename Cpl>() -> kel::lbm::error_or<void>{ - return lbm_main<FloatT,DescT,Cpl,1u>(coupling); + return lbm_main<FloatT,DescT,Cpl,1u>(args); }; - if(coupling == "hlbm"){ - return resolution_resolve.template operator()<method::Hlbm>(); - } - if(coupling == "fplbm"){ - return resolution_resolve.template operator()<method::FpLbm>(); - } - if(coupling == "psm"){ - return resolution_resolve.template operator()<method::Psm>(); - } + auto coupling_resolve = [&]() -> kel::lbm::error_or<void> { + if(coupling.stl_view() == "hlbm"){ + return resolution_resolve.template operator()<method::Hlbm>(); + } else if(coupling.stl_view() == "fplbm"){ + return resolution_resolve.template operator()<method::FpLbm>(); + } else if(coupling.stl_view() == "psm"){ + return resolution_resolve.template operator()<method::Psm>(); + } + }; - /** - * Previously the dispatch lambda had no return on the unmatched path, so - * an unknown coupling fell off the end of a non-void function. - */ - return kel::lbm::make_error<kel::lbm::err::critical>( - std::string{"Unknown coupling '"} + std::string{coupling} - + "', expected one of hlbm, fplbm, psm"); + return coupling_resolve(); + + return kel::lbm::make_error<kel::lbm::err::critical>("Invalid coupling"); } + int main(int argc, char** argv){ auto eov = k_main(argc, argv); if(eov.is_error()){ diff --git a/examples/throat_clog_2d/step.hpp b/examples/throat_clog_2d/step.hpp index 1605cf0..71df661 100644 --- a/examples/throat_clog_2d/step.hpp +++ b/examples/throat_clog_2d/step.hpp @@ -41,7 +41,7 @@ kel::lbm::error_or<void> step( index.at({{i}}).set(idx[i]); } - hlbm_one_part.apply(fields,macros,particles,index,t_i,{0u}); + hlbm_one_part.apply(fields,macros,particles,index,t_i,{1u}); }); }).wait(); @@ -75,7 +75,7 @@ kel::lbm::error_or<void> step( kel::lbm::data<T> scale; { scale.set([&](){ - if(t_i.get() > 1024u){ + if(t_i.get() >= 0u){// > 1024u return 1.0f; } @@ -84,7 +84,7 @@ kel::lbm::error_or<void> step( } { auto y_si = conv.meter_lbm_to_si(as_unit<sch::LbmMeter<T>>(index.at({{1u}}).template cast_to<T>())).handle(); - vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((0.1 / 0.64) * y_si.get() - (0.1 / (0.64*0.64)) * y_si.get() * y_si.get())}).handle() * scale; + vel.at({{0u}}) = conv.velocity_si_to_lbm({static_cast<typename kel::lbm::native_data_type<T>::type>((0.01 / 0.64) * y_si.get() - (0.01 / (0.64*0.64)) * y_si.get() * y_si.get())}).handle(); vel.at({{1u}}) = 0.0f; } return vel; @@ -116,24 +116,9 @@ kel::lbm::error_or<void> step( }); }).wait(); - /** - * Particle step, split so contact lands between the fluid gather and - * the integration. With 20 particles in a converging throat they jam - * against one another, so without a contact pass they would simply - * pass through each other and through the walls. - */ q.submit([&](acpp::sycl::handler& h){ - component<T,Desc,cmpt::ParticleForceReset,encode::Sycl<kel::lbm::enc::Native>> p_reset; - component<T,Desc,cmpt::FpLbmOneParticleGather,encode::Sycl<kel::lbm::enc::Native>> p_gather; - component<T,Desc,cmpt::ParticleIntegrate,encode::Sycl<kel::lbm::enc::Native>> p_integrate; - - /** - * Stiffness is sized so an impact at the inflow speed stays well - * under a tenth of a radius of overlap; see dem_stiffness_for. - */ - dem_params<T> p_dem{2.0e-2, 0.4, 0.0, 0.0}; - kel::lbm::data<sch::FixedArray<sch::UInt64,Desc::D>> p_extent{{dim_x,dim_y}}; - component<T,Desc,cmpt::ParticleContact,encode::Sycl<kel::lbm::enc::Native>> p_contact{p_dem,p_extent}; + kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Desc::D>>> grav; // body acceleration + component<T,Desc,cmpt::FpLbmOneParticle,encode::Sycl<kel::lbm::enc::Native>> fplbm_one_part{grav}; h.parallel_for(acpp::sycl::range<1u>{particle_amount}, [=](acpp::sycl::id<1u> idx){ kel::lbm::data<sch::FixedArray<sch::UInt64,1u>> index; @@ -141,10 +126,7 @@ kel::lbm::error_or<void> step( index.at({{i}}).set(idx[i]); } - p_reset.apply(particles,index,t_i); - p_gather.apply(fields,macros,particles,index,t_i,{4u}); - p_contact.apply(particles,index,t_i,{4u}); - p_integrate.apply(particles,index,t_i,{4u}); + fplbm_one_part.apply(fields,macros,particles,index,t_i,{1u}); }); }).wait(); @@ -220,7 +202,7 @@ kel::lbm::error_or<void> step( index.at({{i}}).set(idx[i]); } - fplbm_one_part.apply(fields,macros,particles,index,t_i,{0u}); + fplbm_one_part.apply(fields,macros,particles,index,t_i,{2u}); }); }).wait(); diff --git a/modules/core/c++/abstract/encode.hpp b/modules/core/c++/abstract/encode.hpp index 571722a..1593656 100644 --- a/modules/core/c++/abstract/encode.hpp +++ b/modules/core/c++/abstract/encode.hpp @@ -25,8 +25,27 @@ struct NativeRaw { struct Json { static constexpr string_literal name = "enc::Json"; }; + +/** + * A packed little endian binary layout. See abstract/forst.hpp. + * + * Every value sits at a computable byte offset, so reading one member costs no + * decoding of the members before it. Fixed size schemas are inline; variable + * length ones spend their slot on dimensions plus an offset and put their bytes + * in a single dynamic layer after the static section. + */ +struct Forst { + static constexpr string_literal name = "enc::Forst"; +}; } // namespace enc +/** + * Declared here rather than in any one codec header so that an encoding does + * not have to include an unrelated one to specialise it. json.hpp and + * forst.hpp are siblings; neither should need the other. + */ +template <typename Schema, typename Encoding> class codec; + #ifndef KEL_LBM_DEFAULT_DATA_ENCODING #define KEL_LBM_DEFAULT_DATA_ENCODING ::kel::lbm::enc::Native #endif diff --git a/modules/core/c++/abstract/forst.hpp b/modules/core/c++/abstract/forst.hpp new file mode 100644 index 0000000..878ad58 --- /dev/null +++ b/modules/core/c++/abstract/forst.hpp @@ -0,0 +1,615 @@ +#pragma once + +#include "data.hpp" +#include "encode.hpp" +#include "schema.hpp" +#include "templates.hpp" + +#include <cstddef> +#include <string_view> +#include <cstdint> +#include <bit> +#include <cstring> +#include <string> +#include <type_traits> +#include <vector> + +/** + * A packed little endian binary encoding for the schema system. + * + * Every value sits at a computable byte offset, so reading one member costs no + * decoding of the members before it. A value is a static section followed by a + * single dynamic layer. Little endian, byte packed, no alignment and no + * padding. Fixed size members sit inline; a variable length member spends its + * static slot on its dimensions as UInt64 each and then an offset, and its + * bytes live in the dynamic layer. + * + * An offset is measured from the byte just past the offset field that holds + * it, so resolving one is a single add with no layer base to track. + * + * This is forstio's Forst format reimplemented against lbm's own schema types + * (lbm's abstract/ is a private reimplementation of forstio's data/codec, so + * the two schema vocabularies are distinct C++ types and nothing could be + * shared), and deliberately cut down to **one dynamic layer**. That makes + * `String` and `Array<Fixed>` expressible while `Array<String>` and + * `Array<Array<...>>` are not -- a static_assert says so rather than letting a + * nested dynamic value mis-encode. + * + * Chunk is not handled here: abstract/ sits below chunk.hpp and must not reach + * up into it. The Chunk layout lives in forst_chunk.hpp, which is what a result + * writer includes. + * + * One limitation shared with enc::Json: a bare `data<Chunk<...>, enc::Forst>` + * is ambiguous, because chunk.hpp already specialises + * `data<Chunk<...>, Encode>` for any encoding and neither specialisation is + * more specialised than the other. Wrap the chunk in a Struct, which is what a + * field always is in practice. + * + * Why it is worth having: a `Struct<Member<Chunk<...>,Key>...>` is entirely + * fixed size, so its encoding *is* the packed field data, byte for byte, with + * no framing overhead and no serialisation pass. Results are written as ASCII + * VTK today, which measures 4-5x larger. + */ +namespace kel { +namespace lbm { + +namespace err { +/// The buffer ended before a value that the layout said was there. +struct forst_truncated { + static constexpr std::string_view description = "Truncated Forst buffer"; + static constexpr bool is_critical = false; +}; + +/// A length or offset in the buffer does not agree with the buffer itself. +struct forst_malformed { + static constexpr std::string_view description = "Malformed Forst buffer"; + static constexpr bool is_critical = false; +}; +} // namespace err + +namespace impl { + +/** + * The layout of a schema. + * + * `layers` is dynamicity depth: 0 for something wholly fixed size, and one + * more for each level of variable length nesting. `static_size` is the bytes + * the schema occupies in a static section -- for a variable length schema that + * is the size of its dimensions plus its offset, not of its contents. + */ +template <typename Schema> struct forst_info { + static_assert(always_false<Schema>, "This schema has no Forst layout"); +}; + +template <> struct forst_info<sch::Void> { + static constexpr uint64_t layers = 0u; + static constexpr uint64_t static_size = 0u; +}; + +template <> struct forst_info<sch::Bool> { + static constexpr uint64_t layers = 0u; + static constexpr uint64_t static_size = 1u; +}; + +template <typename Kind, uint64_t N> struct forst_info<sch::Primitive<Kind, N>> { + static constexpr uint64_t layers = 0u; + static constexpr uint64_t static_size = N; +}; + +/** + * Stored as its storage type, not its interface type. The whole point of a + * mixed precision member is that the narrow form is what occupies space. + */ +template <typename Storage, typename Interface> +struct forst_info<sch::MixedPrecision<Storage, Interface>> { + static constexpr uint64_t layers = forst_info<Storage>::layers; + static constexpr uint64_t static_size = forst_info<Storage>::static_size; +}; + +/// Length, then offset. +template <> struct forst_info<sch::String> { + static constexpr uint64_t layers = 1u; + static constexpr uint64_t static_size = 16u; +}; + +template <typename T, uint64_t... D> struct forst_info<sch::FixedArray<T, D...>> { + static constexpr uint64_t layers = forst_info<T>::layers; + static constexpr uint64_t static_size = + ct_multiply<uint64_t, D...>::value * forst_info<T>::static_size; +}; + +template <typename T, uint64_t... D> struct forst_info<sch::Tensor<T, D...>> { + static constexpr uint64_t layers = forst_info<T>::layers; + static constexpr uint64_t static_size = + ct_multiply<uint64_t, D...>::value * forst_info<T>::static_size; +}; + +/// One UInt64 per dimension, then the offset. +template <typename T, uint64_t Dim> struct forst_info<sch::Array<T, Dim>> { + static constexpr uint64_t layers = 1u + forst_info<T>::layers; + static constexpr uint64_t static_size = 8u * Dim + 8u; +}; + +template <typename... T> struct forst_info<sch::Tuple<T...>> { + template <uint64_t i> static constexpr uint64_t max_layers(){ + if constexpr (i < sizeof...(T)){ + using Ti = typename parameter_pack_type<i, T...>::type; + const uint64_t here = forst_info<Ti>::layers; + const uint64_t rest = max_layers<i + 1u>(); + return (here > rest) ? here : rest; + } + return 0u; + } + + template <uint64_t i> static constexpr uint64_t sum_static(){ + if constexpr (i < sizeof...(T)){ + using Ti = typename parameter_pack_type<i, T...>::type; + return forst_info<Ti>::static_size + sum_static<i + 1u>(); + } + return 0u; + } + + static constexpr uint64_t layers = max_layers<0u>(); + static constexpr uint64_t static_size = sum_static<0u>(); + + /// Byte offset of member i within the static section. + template <uint64_t i> static constexpr uint64_t offset_of(){ + if constexpr (i == 0u){ + return 0u; + } else { + using Prev = typename parameter_pack_type<i - 1u, T...>::type; + return offset_of<i - 1u>() + forst_info<Prev>::static_size; + } + } +}; + +template <typename... T, string_literal... Keys> +struct forst_info<sch::Struct<sch::Member<T, Keys>...>> { + using AsTuple = sch::Tuple<T...>; + + static constexpr uint64_t layers = forst_info<AsTuple>::layers; + static constexpr uint64_t static_size = forst_info<AsTuple>::static_size; + + template <uint64_t i> static constexpr uint64_t offset_of(){ + return forst_info<AsTuple>::template offset_of<i>(); + } +}; + +} // namespace impl + +/** + * The encoded bytes. + * + * Holds its own buffer and copies deeply, like enc::Native. Binary rather than + * text, so unlike data<Schema, enc::Json> this is a byte vector. + */ +template <typename Schema> class data<Schema, enc::Forst> final { +public: + using SchemaType = Schema; + +private: + std::vector<uint8_t> value_; + +public: + data() = default; + + explicit data(std::vector<uint8_t> value__) : value_{std::move(value__)} {} + + void set(std::vector<uint8_t> value__) { value_ = std::move(value__); } + + const std::vector<uint8_t> &bytes() const { return value_; } + std::vector<uint8_t> &bytes() { return value_; } + + const uint8_t *raw() const { return value_.empty() ? nullptr : value_.data(); } + uint8_t *raw() { return value_.empty() ? nullptr : value_.data(); } + + data<sch::UInt64, enc::Native> size() const { return {value_.size()}; } +}; + +namespace impl { + +/** + * Little endian is assumed, never converted. + * + * wasm is little endian by specification and every host this runs on is too. + * A big endian target should fail to build rather than silently produce + * byte-swapped field data that looks almost plausible. + */ +static_assert(std::endian::native == std::endian::little, + "Forst assumes a little endian host"); + +/** + * Bytes a value needs in the dynamic layer. Zero for anything fixed size. + * + * With one layer the contents of a dynamic member are themselves always fixed + * size, so this never has to recurse into a dynamic child's own dynamic part. + */ +template <typename Schema> struct forst_dynamic_size { + static uint64_t apply(const data<Schema, enc::Native> &){ return 0u; } +}; + +template <> struct forst_dynamic_size<sch::String> { + static uint64_t apply(const data<sch::String, enc::Native> &from){ + return from.size().get(); + } +}; + +template <typename T, uint64_t Dim> struct forst_dynamic_size<sch::Array<T, Dim>> { + static uint64_t apply(const data<sch::Array<T, Dim>, enc::Native> &from){ + uint64_t count = 1u; + for(uint64_t i = 0u; i < Dim; ++i){ + count *= from.meta().flat_at(i).get(); + } + return count * forst_info<T>::static_size; + } +}; + +template <typename T, uint64_t... D> +struct forst_dynamic_size<sch::FixedArray<T, D...>> { + static uint64_t apply(const data<sch::FixedArray<T, D...>, enc::Native> &from){ + if constexpr (forst_info<T>::layers == 0u){ + return 0u; + } else { + uint64_t total = 0u; + constexpr uint64_t count = ct_multiply<uint64_t, D...>::value; + for(uint64_t i = 0u; i < count; ++i){ + total += forst_dynamic_size<T>::apply(from.flat_at(i)); + } + return total; + } + } +}; + +template <typename... T, string_literal... Keys> +struct forst_dynamic_size<sch::Struct<sch::Member<T, Keys>...>> { + using Schema = sch::Struct<sch::Member<T, Keys>...>; + + template <uint64_t i> + static uint64_t step(const data<Schema, enc::Native> &from){ + if constexpr (i < sizeof...(T)){ + using Ti = typename parameter_pack_type<i, T...>::type; + return forst_dynamic_size<Ti>::apply(from.template get_at_index<i>()) + + step<i + 1u>(from); + } + return 0u; + } + + static uint64_t apply(const data<Schema, enc::Native> &from){ + return step<0u>(from); + } +}; + +/// Writes a value's static bytes at `at`, appending any contents at `dyn`. +template <typename Schema> struct forst_write; + +template <typename Kind, uint64_t N> struct forst_write<sch::Primitive<Kind, N>> { + static void apply(const data<sch::Primitive<Kind, N>, enc::Native> &from, + uint8_t *base, uint64_t at, uint64_t &){ + auto value = from.get(); + static_assert(sizeof(value) == N, "Primitive storage must be its width"); + std::memcpy(base + at, &value, N); + } +}; + +template <> struct forst_write<sch::Bool> { + static void apply(const data<sch::Bool, enc::Native> &from, uint8_t *base, + uint64_t at, uint64_t &){ + base[at] = from.get() ? 1u : 0u; + } +}; + +template <> struct forst_write<sch::Void> { + static void apply(const data<sch::Void, enc::Native> &, uint8_t *, uint64_t, + uint64_t &){} +}; + +template <> struct forst_write<sch::String> { + static void apply(const data<sch::String, enc::Native> &from, uint8_t *base, + uint64_t at, uint64_t &dyn){ + const uint64_t length = from.size().get(); + std::memcpy(base + at, &length, 8u); + + // Measured from just past the offset field, so resolving it is one add. + const uint64_t offset = dyn - (at + 16u); + std::memcpy(base + at + 8u, &offset, 8u); + + if(length > 0u){ + std::memcpy(base + dyn, from.stl_view().data(), length); + } + dyn += length; + } +}; + +template <typename T, uint64_t Dim> struct forst_write<sch::Array<T, Dim>> { + static void apply(const data<sch::Array<T, Dim>, enc::Native> &from, + uint8_t *base, uint64_t at, uint64_t &dyn){ + static_assert(forst_info<T>::layers == 0u, + "One dynamic layer only: an Array element must be fixed size. " + "Array<String> and Array<Array<...>> need a second layer."); + + uint64_t count = 1u; + for(uint64_t i = 0u; i < Dim; ++i){ + const uint64_t extent = from.meta().flat_at(i).get(); + std::memcpy(base + at + 8u * i, &extent, 8u); + count *= extent; + } + + const uint64_t offset = dyn - (at + forst_info<sch::Array<T, Dim>>::static_size); + std::memcpy(base + at + 8u * Dim, &offset, 8u); + + // Elements are fixed size, so they go straight into the layer with no + // dynamic part of their own. + uint64_t unused = 0u; + const uint64_t stride = forst_info<T>::static_size; + for(uint64_t i = 0u; i < count; ++i){ + forst_write<T>::apply(from.flat_at(i), base, dyn + i * stride, unused); + } + dyn += count * stride; + } +}; + +template <typename T, uint64_t... D> struct forst_write<sch::FixedArray<T, D...>> { + static void apply(const data<sch::FixedArray<T, D...>, enc::Native> &from, + uint8_t *base, uint64_t at, uint64_t &dyn){ + constexpr uint64_t count = ct_multiply<uint64_t, D...>::value; + constexpr uint64_t stride = forst_info<T>::static_size; + for(uint64_t i = 0u; i < count; ++i){ + forst_write<T>::apply(from.flat_at(i), base, at + i * stride, dyn); + } + } +}; + +template <typename T, uint64_t... D> struct forst_write<sch::Tensor<T, D...>> { + static void apply(const data<sch::Tensor<T, D...>, enc::Native> &from, + uint8_t *base, uint64_t at, uint64_t &dyn){ + constexpr uint64_t count = ct_multiply<uint64_t, D...>::value; + constexpr uint64_t stride = forst_info<T>::static_size; + for(uint64_t i = 0u; i < count; ++i){ + forst_write<T>::apply(from.flat_at(i), base, at + i * stride, dyn); + } + } +}; +template <typename... T, string_literal... Keys> +struct forst_write<sch::Struct<sch::Member<T, Keys>...>> { + using Schema = sch::Struct<sch::Member<T, Keys>...>; + using Info = forst_info<Schema>; + + template <uint64_t i> + static void step(const data<Schema, enc::Native> &from, uint8_t *base, + uint64_t at, uint64_t &dyn){ + if constexpr (i < sizeof...(T)){ + using Ti = typename parameter_pack_type<i, T...>::type; + forst_write<Ti>::apply(from.template get_at_index<i>(), base, + at + Info::template offset_of<i>(), dyn); + step<i + 1u>(from, base, at, dyn); + } + } + + static void apply(const data<Schema, enc::Native> &from, uint8_t *base, + uint64_t at, uint64_t &dyn){ + step<0u>(from, base, at, dyn); + } +}; + + +/** + * Reads a value's static bytes at `at`. + * + * Every read is bounds checked against the buffer rather than trusted: the + * bytes may have come off a network from a peer that is older, newer, or + * hostile, and a bad offset would otherwise be an out of bounds read. + */ +template <typename Schema> struct forst_read; + +template <typename Kind, uint64_t N> struct forst_read<sch::Primitive<Kind, N>> { + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<sch::Primitive<Kind, N>, enc::Native> &to){ + if(at + N > size){ + return make_error<err::forst_truncated>(); + } + typename native_data_type<sch::Primitive<Kind, N>>::type value{}; + std::memcpy(&value, base + at, N); + to.set(value); + return make_void(); + } +}; + +template <> struct forst_read<sch::Bool> { + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<sch::Bool, enc::Native> &to){ + if(at + 1u > size){ + return make_error<err::forst_truncated>(); + } + to.set(base[at] != 0u); + return make_void(); + } +}; + +template <> struct forst_read<sch::Void> { + static error_or<void> apply(const uint8_t *, uint64_t, uint64_t, + data<sch::Void, enc::Native> &){ + return make_void(); + } +}; + +template <> struct forst_read<sch::String> { + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<sch::String, enc::Native> &to){ + if(at + 16u > size){ + return make_error<err::forst_truncated>(); + } + + uint64_t length = 0u; + uint64_t offset = 0u; + std::memcpy(&length, base + at, 8u); + std::memcpy(&offset, base + at + 8u, 8u); + + const uint64_t start = at + 16u + offset; + // Checked as a sum that cannot wrap, so a hostile length plus a + // hostile offset cannot alias back into the static section. + if(start > size or length > size - start){ + return make_error<err::forst_malformed>(); + } + + to.set(std::string{reinterpret_cast<const char *>(base + start), + static_cast<std::size_t>(length)}); + return make_void(); + } +}; + +template <typename T, uint64_t Dim> struct forst_read<sch::Array<T, Dim>> { + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<sch::Array<T, Dim>, enc::Native> &to){ + static_assert(forst_info<T>::layers == 0u, + "One dynamic layer only: an Array element must be fixed size."); + + constexpr uint64_t slot = forst_info<sch::Array<T, Dim>>::static_size; + if(at + slot > size){ + return make_error<err::forst_truncated>(); + } + + index_data<Dim> extents; + uint64_t count = 1u; + for(uint64_t i = 0u; i < Dim; ++i){ + uint64_t extent = 0u; + std::memcpy(&extent, base + at + 8u * i, 8u); + extents.flat_at(i).set(extent); + + // Reject an extent product that would overflow before it is used + // to size an allocation. + if(extent != 0u and count > (~uint64_t{0u}) / extent){ + return make_error<err::forst_malformed>(); + } + count *= extent; + } + + uint64_t offset = 0u; + std::memcpy(&offset, base + at + 8u * Dim, 8u); + + constexpr uint64_t stride = forst_info<T>::static_size; + const uint64_t start = at + slot + offset; + if(start > size or (stride != 0u and count > (size - start) / stride)){ + return make_error<err::forst_malformed>(); + } + + to = data<sch::Array<T, Dim>, enc::Native>{extents}; + for(uint64_t i = 0u; i < count; ++i){ + auto eov = forst_read<T>::apply(base, size, start + i * stride, + to.flat_at(i)); + if(eov.is_error()){ + return eov; + } + } + return make_void(); + } +}; + +template <typename T, uint64_t... D> struct forst_read<sch::FixedArray<T, D...>> { + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<sch::FixedArray<T, D...>, enc::Native> &to){ + constexpr uint64_t count = ct_multiply<uint64_t, D...>::value; + constexpr uint64_t stride = forst_info<T>::static_size; + for(uint64_t i = 0u; i < count; ++i){ + auto eov = forst_read<T>::apply(base, size, at + i * stride, + to.flat_at(i)); + if(eov.is_error()){ + return eov; + } + } + return make_void(); + } +}; + +template <typename T, uint64_t... D> struct forst_read<sch::Tensor<T, D...>> { + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<sch::Tensor<T, D...>, enc::Native> &to){ + constexpr uint64_t count = ct_multiply<uint64_t, D...>::value; + constexpr uint64_t stride = forst_info<T>::static_size; + for(uint64_t i = 0u; i < count; ++i){ + auto eov = forst_read<T>::apply(base, size, at + i * stride, + to.flat_at(i)); + if(eov.is_error()){ + return eov; + } + } + return make_void(); + } +}; + +template <typename... T, string_literal... Keys> +struct forst_read<sch::Struct<sch::Member<T, Keys>...>> { + using Schema = sch::Struct<sch::Member<T, Keys>...>; + using Info = forst_info<Schema>; + + template <uint64_t i> + static error_or<void> step(const uint8_t *base, uint64_t size, uint64_t at, + data<Schema, enc::Native> &to){ + if constexpr (i < sizeof...(T)){ + using Ti = typename parameter_pack_type<i, T...>::type; + auto eov = forst_read<Ti>::apply( + base, size, at + Info::template offset_of<i>(), + to.template get_at_index<i>()); + if(eov.is_error()){ + return eov; + } + return step<i + 1u>(base, size, at, to); + } + return make_void(); + } + + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<Schema, enc::Native> &to){ + return step<0u>(base, size, at, to); + } +}; + +} // namespace impl + +/** + * Translates between a native value and its Forst bytes. + */ +template <typename Schema> class codec<Schema, enc::Forst> final { +public: + static_assert(impl::forst_info<Schema>::layers <= 1u, + "Forst here carries exactly one dynamic layer. A schema needing more " + "-- Array<String>, Array<Array<...>> -- has to be flattened."); + + codec() = default; + + KEL_FORBID_COPY(codec); + KEL_DEFAULT_MOVE(codec); + + error_or<void> encode(const data<Schema, enc::Native> &from, + data<Schema, enc::Forst> &to){ + constexpr uint64_t static_size = impl::forst_info<Schema>::static_size; + const uint64_t dynamic_size = impl::forst_dynamic_size<Schema>::apply(from); + + std::vector<uint8_t> buffer; + try { + buffer.resize(static_cast<std::size_t>(static_size + dynamic_size)); + } catch (const std::exception &e){ + (void) e; + return make_error<err::out_of_memory>(); + } + + uint64_t dyn = static_size; + if(not buffer.empty()){ + impl::forst_write<Schema>::apply(from, buffer.data(), 0u, dyn); + } + + to.set(std::move(buffer)); + return make_void(); + } + + error_or<void> decode(const data<Schema, enc::Forst> &from, + data<Schema, enc::Native> &to){ + const uint64_t size = from.size().get(); + if(size < impl::forst_info<Schema>::static_size){ + return make_error<err::forst_truncated>(); + } + return impl::forst_read<Schema>::apply(from.raw(), size, 0u, to); + } +}; + +} // namespace lbm +} // namespace kel diff --git a/modules/core/c++/chunk.hpp b/modules/core/c++/chunk.hpp index 126add8..8edd6ff 100644 --- a/modules/core/c++/chunk.hpp +++ b/modules/core/c++/chunk.hpp @@ -97,6 +97,16 @@ public: return values_.flat_data(); } + /** + * The const counterpart data<FixedArray> has always had. Without it a + * const-correct reader -- a Forst encode, or a writer taking + * `const data<Sch>&` the way write_vtk_file does -- cannot reach the + * bytes at all. + */ + auto flat_data() const { + return values_.flat_data(); + } + static constexpr auto flat_size() { return data<InnerSchema,Encode>::flat_dims(); } diff --git a/modules/core/c++/forst_chunk.hpp b/modules/core/c++/forst_chunk.hpp new file mode 100644 index 0000000..7d4c105 --- /dev/null +++ b/modules/core/c++/forst_chunk.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include "abstract/forst.hpp" +#include "chunk.hpp" + +/** + * The Forst layout for Chunk. + * + * Separate from abstract/forst.hpp because abstract/ sits below chunk.hpp and + * must not reach up into it. Include this rather than the abstract header when + * encoding fields. + * + * A Chunk is its InnerSchema -- a FixedArray widened by the ghost ring -- so it + * is entirely fixed size and encodes to exactly the bytes already sitting in + * flat_data(). That is what makes sending a field snapshot free of any + * serialisation pass or second buffer, which matters most in a browser tab + * where a second 18 MiB allocation may simply not be available. + * + * The encoded extents are `Sides + 2*Ghost`, not the logical sides. A reader + * without Ghost cannot index the buffer, which is why the result manifest + * carries it. + */ +namespace kel { +namespace lbm { +namespace impl { + +template <typename T, uint64_t Ghost, uint64_t... Sides> +struct forst_info<sch::Chunk<T, Ghost, Sides...>> { + using Inner = typename sch::Chunk<T, Ghost, Sides...>::InnerSchema; + static constexpr uint64_t layers = forst_info<Inner>::layers; + static constexpr uint64_t static_size = forst_info<Inner>::static_size; +}; + +template <typename T, uint64_t Ghost, uint64_t... Sides> +struct forst_write<sch::Chunk<T, Ghost, Sides...>> { + using Schema = sch::Chunk<T, Ghost, Sides...>; + using Value = typename Schema::StoredValueSchema; + + static void apply(const data<Schema, enc::Native> &from, uint8_t *base, + uint64_t at, uint64_t &dyn){ + const uint64_t count = data<Schema, enc::Native>::flat_size().get(); + constexpr uint64_t stride = forst_info<Value>::static_size; + const auto *elements = from.flat_data(); + for(uint64_t i = 0u; i < count; ++i){ + forst_write<Value>::apply(elements[i], base, at + i * stride, dyn); + } + } +}; + +template <typename T, uint64_t Ghost, uint64_t... Sides> +struct forst_read<sch::Chunk<T, Ghost, Sides...>> { + using Schema = sch::Chunk<T, Ghost, Sides...>; + using Value = typename Schema::StoredValueSchema; + + static error_or<void> apply(const uint8_t *base, uint64_t size, uint64_t at, + data<Schema, enc::Native> &to){ + const uint64_t count = data<Schema, enc::Native>::flat_size().get(); + constexpr uint64_t stride = forst_info<Value>::static_size; + auto *elements = to.flat_data(); + for(uint64_t i = 0u; i < count; ++i){ + auto eov = forst_read<Value>::apply(base, size, at + i * stride, + elements[i]); + if(eov.is_error()){ + return eov; + } + } + return make_void(); + } +}; + +} // namespace impl +} // namespace lbm +} // namespace kel diff --git a/modules/core/c++/fplbm.hpp b/modules/core/c++/fplbm.hpp index bc99da5..780572b 100644 --- a/modules/core/c++/fplbm.hpp +++ b/modules/core/c++/fplbm.hpp @@ -2,7 +2,6 @@ #include "common.hpp" #include "particle/particle.hpp" -#include "particle/integrate.hpp" #include "iterator.hpp" namespace kel { @@ -13,7 +12,6 @@ struct FpLbm {}; struct FpLbmOneParticle {}; struct FpLbmMomentumOneParticle {}; struct FpLbmOneParticleNoVelocity{}; -struct FpLbmOneParticleGather {}; } namespace method { struct FpLbm {}; @@ -101,7 +99,10 @@ public: // We already computed this in reset // compute_rho_u<T,Descriptor>(dfs,rho,vel); - mom = mom + force * half; + + // ------------------------- + // mom = mom + force * half; + // ------------------------- auto vel = mom / rho; auto eq = equilibrium<T,Descriptor>(rho,vel); @@ -140,11 +141,10 @@ public: }; template<typename T, typename Descriptor, typename Encode> -class component<T, Descriptor, cmpt::FpLbmOneParticleGather, Encode> final { +class component<T, Descriptor, cmpt::FpLbmOneParticle, Encode> final { public: - using Component = cmpt::FpLbmOneParticleGather; + using Component = cmpt::FpLbmOneParticle; private: - // An acceleration, as the comment above says -- now typed as one. kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Descriptor::D>>> body_acceleration_; public: component(): @@ -175,28 +175,31 @@ public: auto parts = pg.template get<"particles">(); auto parts_size = parts.meta().at({0u}); - auto& pi = parts.at(index); + auto& pi = parts.at(index); auto& pirb = pi.template get<"rigid_body">(); + // Unwrapped once: the sweep works in raw lattice offsets. auto& pirb_pos = pirb.template get<"position">().handle(); auto& pirb_pos_old = pirb.template get<"position_old">().handle(); - auto& p_rad = pi.template get<"collision">().template get<"radius">().handle(); + auto& p_coll = pg.template get<"collision">().at({}); + auto& p_rad = p_coll.template get<"radius">().handle(); - kel::lbm::data<typename particle_aabb<typename ParticleSchema::ValueType>::AABB> aabb; - if(not particle_aabb<typename ParticleSchema::ValueType>::try_calculate(pg,index,mom_f.meta(),aabb)){ + auto eo_aabb = particle_aabb<typename ParticleSchema::ValueType>::calculate(pg,{{0u}},mom_f.meta()); + if(eo_aabb.is_error()){ return; } + auto& aabb = eo_aabb.get_value(); kel::lbm::data<sch::Scalar<T>> three; three.at({}).set(3.0f); - kel::lbm::data<sch::Scalar<T>> two; - two.at({}).set(2.0f); - kel::lbm::data<sch::Scalar<T>> one; one.at({}) = 1.0f; + + kel::lbm::data<sch::Scalar<T>> k; + k.at({}) = 1.0f; kel::lbm::data<sch::Scalar<T>> eps; eps.at({}) = 1.5f; @@ -222,51 +225,27 @@ public: // vel_s is technically time the density of the particle? // force = body_acceleration_ * por + ( vel_s - mom) * two * flip_por; - force = ( vel_s - mom) * two * flip_por; + // force = ( vel_s - mom) * eps * flip_por; + force = ( vel_s - mom) * k * flip_por; // force = (vel_s - mom) * flip_por / (flip_por / two + one); force_p = force_p - force; }, aabb.template get<"a">(), aabb.template get<"b">()); - /** - * Accumulate rather than assign, so a contact pass can add to this - * before cmpt::ParticleIntegrate turns the total into motion. - */ - auto& pirb_force = pirb.template get<"force">(); - pirb_force = pirb_force + force_p; - } -}; - -/** - * The original fused one-particle step, kept so existing call sites keep - * working. The body acceleration rides straight through to the integrator; - * it used to be called global_force_, which it never was -- the integrator - * takes an acceleration, not a force. - */ -template<typename T, typename Descriptor, typename Encode> -class component<T, Descriptor, cmpt::FpLbmOneParticle, Encode> final { -public: - using Component = cmpt::FpLbmOneParticle; -private: - // An acceleration, as the comment above says -- now typed as one. - kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Descriptor::D>>> body_acceleration_; -public: - component(): - body_acceleration_{} - {} - component(kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Descriptor::D>>> body_acceleration__): - body_acceleration_{body_acceleration__} - {} - - template<typename CellFieldSchema, typename MacroFieldSchema, typename ParticleSchema> - void apply(const kel::lbm::data<CellFieldSchema, Encode>& field, const kel::lbm::data<MacroFieldSchema,Encode>& macros, const kel::lbm::data<ParticleSchema,Encode>& pg, kel::lbm::data<sch::FixedArray<sch::UInt64,1u>> index, kel::lbm::data<sch::UInt64> time_step, kel::lbm::data<sch::UInt64> sub_steps) const { - component<T,Descriptor,cmpt::ParticleForceReset,Encode> reset; - component<T,Descriptor,cmpt::FpLbmOneParticleGather,Encode> gather; - component<T,Descriptor,cmpt::ParticleIntegrate,Encode> integrate{body_acceleration_}; + auto& pirb_acc = pirb.template get<"acceleration">(); + // Mass moved onto the particle; the quotient is untyped because force + // and mass have no units, so the acceleration unit is attached here. + pirb_acc = kel::lbm::data<sch::LbmAcceleration<sch::Vector<T,Descriptor::D>>>{ + force_p / (pirb.template get<"mass">().at({}))} + + body_acceleration_; - reset.apply(pg,index,time_step); - gather.apply(field,macros,pg,index,time_step,sub_steps); - integrate.apply(pg,index,time_step,sub_steps); + if(sub_steps.get() > 0u){ + // The sub step length in lattice time units: one full step split sss ways. + kel::lbm::data<sch::LbmSecond<sch::Scalar<T>>> sss_inv{one / sss}; + for(kel::lbm::data<sch::UInt64> i{0u}; i < sub_steps; ++i){ + verlet_step_lambda<T,Descriptor::D>(pi,sss_inv); + } + } } }; } diff --git a/modules/core/c++/write_vtk.hpp b/modules/core/c++/write_vtk.hpp index 028a26f..4c67b74 100644 --- a/modules/core/c++/write_vtk.hpp +++ b/modules/core/c++/write_vtk.hpp @@ -1,7 +1,6 @@ #pragma once #include "abstract/error.hpp" - #include "abstract/data.hpp" #include "abstract/data_math.hpp" @@ -47,7 +46,7 @@ struct lbm_vtk_writer<sch::FixedArray<T,D>> { static kel::lbm::error_or<void> apply(std::ostream& vtk_file, const kel::lbm::data<sch::FixedArray<T,D>>& field){ kel::lbm::data<sch::FixedArray<sch::UInt64,D>> index; for(kel::lbm::data<sch::UInt64> it{0}; it.get() < D; ++it){ - index.at({0u}).set(0u); + index.at(it).set(0u); } // vtk_file<<"VECTORS "<<name<<" float\n"; diff --git a/modules/core/tests/forst.cpp b/modules/core/tests/forst.cpp new file mode 100644 index 0000000..2384126 --- /dev/null +++ b/modules/core/tests/forst.cpp @@ -0,0 +1,361 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/abstract/forst.hpp" +#include "../c++/forst_chunk.hpp" + +#include <cstring> +#include <string> + +namespace { +namespace sch { +using namespace kel::lbm::sch; +} +using namespace kel::lbm; + +template <typename Schema> +error_or<void> round_trip(const data<Schema> &from, data<Schema> &to){ + codec<Schema, enc::Forst> c; + + data<Schema, enc::Forst> bytes; + auto eov = c.encode(from, bytes); + if(eov.is_error()){ + return eov; + } + return c.decode(bytes, to); +} + +template <typename Schema> +uint64_t encoded_size(const data<Schema> &from){ + codec<Schema, enc::Forst> c; + data<Schema, enc::Forst> bytes; + auto eov = c.encode(from, bytes); + (void) eov; + return bytes.size().get(); +} + +SAW_TEST("Forst / layout traits match the declared table"){ + using namespace kel::lbm::impl; + + static_assert(forst_info<sch::UInt8>::static_size == 1u, "u8"); + static_assert(forst_info<sch::Float32>::static_size == 4u, "f32"); + static_assert(forst_info<sch::Float64>::static_size == 8u, "f64"); + static_assert(forst_info<sch::UInt8>::layers == 0u, "primitives are fixed"); + + static_assert(forst_info<sch::String>::static_size == 16u, "length + offset"); + static_assert(forst_info<sch::String>::layers == 1u, "a string is dynamic"); + + static_assert(forst_info<sch::Array<sch::UInt64, 1u>>::static_size == 16u, + "one extent + offset"); + static_assert(forst_info<sch::Array<sch::UInt64, 2u>>::static_size == 24u, + "two extents + offset"); + + static_assert(forst_info<sch::FixedArray<sch::Float32, 9u>>::static_size == 36u, + "a fixed array is its elements inline"); + static_assert(forst_info<sch::FixedArray<sch::Float32, 9u>>::layers == 0u, + "and stays fixed"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +/** + * The property the whole design rests on: an array of a dynamic element needs + * a second layer, and must therefore be rejected rather than mis-encoded. + */ +SAW_TEST("Forst / nesting depth is counted, so two layers are detectable"){ + using namespace kel::lbm::impl; + + static_assert(forst_info<sch::Array<sch::String, 1u>>::layers == 2u, + "Array<String> is two layers and must not pass the codec's assert"); + static_assert( + forst_info<sch::Array<sch::Array<sch::UInt64,1u>, 1u>>::layers == 2u, + "Array<Array<...>> likewise"); + + using Flat = sch::Struct< + sch::Member<sch::String, "a">, + sch::Member<sch::Array<sch::UInt64, 1u>, "b"> + >; + static_assert(forst_info<Flat>::layers == 1u, + "Several dynamic members side by side are still one layer"); + + // And the thing the result manifest depends on: an array of a *fixed* + // struct stays at one layer. + using FixedElem = sch::Struct< + sch::Member<sch::FixedArray<sch::UInt8, 32u>, "name">, + sch::Member<sch::UInt64, "bytes"> + >; + static_assert(forst_info<FixedElem>::layers == 0u, "fixed struct"); + static_assert(forst_info<sch::Array<FixedElem, 1u>>::layers == 1u, + "an array of fixed structs is what keeps the manifest flat"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +SAW_TEST("Forst / primitives round trip exactly"){ + { + data<sch::Float32> from; from.set(-1.5f); + data<sch::Float32> to; + SAW_EXPECT_EOV(round_trip(from, to), "f32"); + SAW_EXPECT(to.get() == -1.5f, "f32 survives bit for bit"); + } + { + data<sch::Float64> from; from.set(1.0 / 3.0); + data<sch::Float64> to; + SAW_EXPECT_EOV(round_trip(from, to), "f64"); + SAW_EXPECT(to.get() == 1.0 / 3.0, "f64 survives bit for bit"); + } + { + // The edges, where a width or sign mistake would show. + data<sch::UInt64> from; from.set(~uint64_t{0u}); + data<sch::UInt64> to; + SAW_EXPECT_EOV(round_trip(from, to), "u64"); + SAW_EXPECT(to.get() == ~uint64_t{0u}, "u64 max survives"); + } + { + data<sch::Int32> from; from.set(-2147483647 - 1); + data<sch::Int32> to; + SAW_EXPECT_EOV(round_trip(from, to), "i32"); + SAW_EXPECT(to.get() == -2147483647 - 1, "i32 min survives"); + } + { + data<sch::Bool> from; from.set(true); + data<sch::Bool> to; + SAW_EXPECT_EOV(round_trip(from, to), "bool"); + SAW_EXPECT(to.get(), "bool survives"); + SAW_EXPECT(encoded_size(from) == 1u, "and costs one byte"); + } +} + +SAW_TEST("Forst / strings round trip, including the empty one"){ + { + data<sch::String> from{std::string{"velocity"}}; + data<sch::String> to; + SAW_EXPECT_EOV(round_trip(from, to), "string"); + SAW_EXPECT(to.stl_view() == "velocity", "contents survive"); + SAW_EXPECT(encoded_size(from) == 16u + 8u, "slot plus contents"); + } + { + data<sch::String> from{std::string{""}}; + data<sch::String> to{std::string{"not empty"}}; + SAW_EXPECT_EOV(round_trip(from, to), "empty string"); + SAW_EXPECT(to.stl_view().empty(), "an empty string stays empty"); + SAW_EXPECT(encoded_size(from) == 16u, "and costs only its slot"); + } + { + // Embedded NULs and high bytes: this is a byte string, not C text. + std::string raw{"a\0b\xff", 4u}; + data<sch::String> from{raw}; + data<sch::String> to; + SAW_EXPECT_EOV(round_trip(from, to), "binary string"); + SAW_EXPECT(to.stl_string() == raw, "NULs and high bytes survive"); + } +} + +SAW_TEST("Forst / several dynamic members share the one layer"){ + using Schema = sch::Struct< + sch::Member<sch::String, "first">, + sch::Member<sch::UInt32, "middle">, + sch::Member<sch::String, "last"> + >; + + data<Schema> from; + from.get<"first">().set(std::string{"alpha"}); + from.get<"middle">().set(42u); + from.get<"last">().set(std::string{"omega"}); + + data<Schema> to; + SAW_EXPECT_EOV(round_trip(from, to), "struct with two strings"); + + SAW_EXPECT(to.get<"first">().stl_view() == "alpha", "first survives"); + SAW_EXPECT(to.get<"middle">().get() == 42u, "the fixed member between them survives"); + SAW_EXPECT(to.get<"last">().stl_view() == "omega", "and so does the second string"); + + // 16 + 4 + 16 static, then 5 + 5 of contents. + SAW_EXPECT(encoded_size(from) == 36u + 10u, "no padding anywhere"); +} + +SAW_TEST("Forst / an array of fixed structs round trips"){ + using Elem = sch::Struct< + sch::Member<sch::UInt32, "id">, + sch::Member<sch::Float32, "value"> + >; + using Schema = sch::Struct<sch::Member<sch::Array<Elem, 1u>, "items">>; + + data<Schema> from; + for(uint32_t i = 0u; i < 4u; ++i){ + data<Elem> elem; + elem.get<"id">().set(i); + elem.get<"value">().set(static_cast<float>(i) * 0.5f); + auto eov = from.get<"items">().add(std::move(elem)); + SAW_EXPECT_EOV(eov, "array grows"); + } + + data<Schema> to; + SAW_EXPECT_EOV(round_trip(from, to), "array of structs"); + + SAW_EXPECT(to.get<"items">().meta().flat_at(0u).get() == 4u, + "the extent survives"); + for(uint64_t i = 0u; i < 4u; ++i){ + SAW_EXPECT(to.get<"items">().at({{i}}).get<"id">().get() == i, + "element ids survive"); + SAW_EXPECT(to.get<"items">().at({{i}}).get<"value">().get() + == static_cast<float>(i) * 0.5f, "element values survive"); + } + + // 16 static for the slot, 8 per element. + SAW_EXPECT(encoded_size(from) == 16u + 4u * 8u, "exact, unpadded"); +} + +/** + * The claim the whole encoding choice rests on: a struct of chunks is entirely + * fixed size, so its Forst form is the packed field data itself. If this ever + * stops holding, sending results costs a serialisation pass and a second + * buffer -- in a browser tab, possibly one that does not exist. + */ +SAW_TEST("Forst / a struct of chunks is exactly its raw bytes"){ + constexpr uint64_t dim_x = 16u; + constexpr uint64_t dim_y = 8u; + + using Velocity = sch::Chunk<sch::Vector<sch::Float32, 2u>, 0u, dim_x, dim_y>; + using Density = sch::Chunk<sch::Scalar<sch::Float32>, 0u, dim_x, dim_y>; + using Macro = sch::Struct< + sch::Member<Velocity, "velocity">, + sch::Member<Density, "density"> + >; + + // The standard-layout assumption the zero-copy path depends on. + static_assert(sizeof(data<sch::Vector<sch::Float32, 2u>>) == 2u * sizeof(float), + "A Vector must be its components with nothing around them"); + static_assert(sizeof(data<sch::Scalar<sch::Float32>>) == sizeof(float), + "A Scalar must be its component with nothing around it"); + + static_assert(kel::lbm::impl::forst_info<Macro>::layers == 0u, + "A struct of chunks has no dynamic part at all"); + + constexpr uint64_t expected = dim_x * dim_y * (2u + 1u) * 4u; + static_assert(kel::lbm::impl::forst_info<Macro>::static_size == expected, + "and occupies exactly the field data"); + + data<Macro> from; + for(uint64_t y = 0u; y < dim_y; ++y){ + for(uint64_t x = 0u; x < dim_x; ++x){ + auto &vel = from.get<"velocity">().at({{x, y}}); + vel.at({{0u}}).set(static_cast<float>(x)); + vel.at({{1u}}).set(static_cast<float>(y)); + from.get<"density">().at({{x, y}}).at({}).set( + static_cast<float>(x * dim_y + y)); + } + } + + codec<Macro, enc::Forst> c; + data<Macro, enc::Forst> bytes; + SAW_EXPECT_EOV(c.encode(from, bytes), "encode"); + + SAW_EXPECT(bytes.size().get() == expected, + "The encoding adds not one byte of framing"); + + // And the bytes are the buffer, not merely the same size as it. + const auto *velocity_raw = reinterpret_cast<const uint8_t *>( + from.get<"velocity">().flat_data()); + SAW_EXPECT(std::memcmp(bytes.raw(), velocity_raw, dim_x * dim_y * 2u * 4u) == 0, + "The velocity chunk encodes to its own memory, byte for byte"); + + const auto *density_raw = reinterpret_cast<const uint8_t *>( + from.get<"density">().flat_data()); + SAW_EXPECT(std::memcmp(bytes.raw() + dim_x * dim_y * 2u * 4u, density_raw, + dim_x * dim_y * 4u) == 0, + "and the density chunk follows it with no gap"); + + data<Macro> to; + SAW_EXPECT_EOV(c.decode(bytes, to), "decode"); + SAW_EXPECT(to.get<"velocity">().at({{3u, 5u}}).at({{1u}}).get() == 5.0f, + "values come back where they went in"); + SAW_EXPECT(to.get<"density">().at({{3u, 5u}}).at({}).get() + == static_cast<float>(3u * dim_y + 5u), "for every member"); +} + +SAW_TEST("Forst / a chunk's ghost ring is part of the encoding"){ + // Ghost=1 widens each side by two, and flat_data covers the ring. A reader + // that assumed the logical extents would be short by the whole border. + using Df = sch::Chunk<sch::FixedArray<sch::Float32, 9u>, 1u, 8u, 4u>; + + constexpr uint64_t stored = (8u + 2u) * (4u + 2u); + static_assert(kel::lbm::impl::forst_info<Df>::static_size == stored * 9u * 4u, + "The encoded size is the padded extents, not the logical ones"); + + // Wrapped in a Struct rather than encoded bare: chunk.hpp already + // specialises data<Chunk<...>, Encode> for any encoding, so a top level + // data<Chunk, enc::Forst> is ambiguous against it. enc::Json has the same + // clash. A field is always a member of a struct in practice. + using Field = sch::Struct<sch::Member<Df, "dfs">>; + + data<Field> from; + from.get<"dfs">().at({{0u, 0u}}).at({3u}).set(7.0f); + + data<Field> to; + SAW_EXPECT_EOV(round_trip(from, to), "df chunk"); + SAW_EXPECT(to.get<"dfs">().at({{0u, 0u}}).at({3u}).get() == 7.0f, + "interior survives"); +} + +SAW_TEST("Forst / truncated input is rejected rather than read past"){ + using Schema = sch::Struct< + sch::Member<sch::String, "name">, + sch::Member<sch::UInt64, "count"> + >; + + data<Schema> from; + from.get<"name">().set(std::string{"something"}); + from.get<"count">().set(5u); + + codec<Schema, enc::Forst> c; + data<Schema, enc::Forst> bytes; + SAW_EXPECT_EOV(c.encode(from, bytes), "encode"); + + // Cut the contents off and the string's length no longer fits the buffer. + auto truncated = bytes.bytes(); + truncated.resize(truncated.size() - 4u); + data<Schema, enc::Forst> damaged{std::move(truncated)}; + + data<Schema> to; + SAW_EXPECT(c.decode(damaged, to).is_error(), + "A string reaching past the end must be an error, not a read"); + + // Cut below the static section and it must fail before indexing anything. + data<Schema, enc::Forst> stub{std::vector<uint8_t>(4u, 0u)}; + SAW_EXPECT(c.decode(stub, to).is_error(), + "A buffer shorter than the static section must be rejected up front"); + SAW_EXPECT(c.decode(data<Schema, enc::Forst>{}, to).is_error(), + "and so must an empty one"); +} + +SAW_TEST("Forst / a hostile offset cannot escape the buffer"){ + using Schema = sch::Struct<sch::Member<sch::String, "name">>; + + data<Schema> from; + from.get<"name">().set(std::string{"abc"}); + + codec<Schema, enc::Forst> c; + data<Schema, enc::Forst> bytes; + SAW_EXPECT_EOV(c.encode(from, bytes), "encode"); + + // A length near the top of the range must not wrap when added to a start + // that is itself inside the buffer. + { + auto tampered = bytes.bytes(); + const uint64_t huge = ~uint64_t{0u} - 8u; + std::memcpy(tampered.data(), &huge, 8u); + data<Schema, enc::Forst> evil{std::move(tampered)}; + data<Schema> to; + SAW_EXPECT(c.decode(evil, to).is_error(), "An absurd length is rejected"); + } + // An offset pointing far outside likewise. + { + auto tampered = bytes.bytes(); + const uint64_t far = ~uint64_t{0u} / 2u; + std::memcpy(tampered.data() + 8u, &far, 8u); + data<Schema, enc::Forst> evil{std::move(tampered)}; + data<Schema> to; + SAW_EXPECT(c.decode(evil, to).is_error(), "An absurd offset is rejected"); + } +} +} diff --git a/examples/cavity_2d/.nix/derivation.nix b/modules/swarm/.nix/derivation.nix index d0660e9..afd07ef 100644 --- a/examples/cavity_2d/.nix/derivation.nix +++ b/modules/swarm/.nix/derivation.nix @@ -9,7 +9,7 @@ }: stdenv.mkDerivation { - pname = pname + "-examples-" + "cavity_2d"; + pname = "${pname}-swarm"; inherit version; src = ./..; @@ -18,15 +18,20 @@ stdenv.mkDerivation { clang-tools ]; + # The module itself is forstio free -- it is schemas and arithmetic. forstio + # is here only for the test runner, exactly as in modules/core. buildInputs = [ forstio.core - forstio.async - forstio.codec - forstio.codec-unit - forstio.codec-json kel.lbm.core ]; + doCheck = true; + + checkPhase = '' + scons test + ./bin/tests + ''; + preferLocalBuild = true; outputs = [ "out" "dev" ]; diff --git a/examples/cavity_2d/SConstruct b/modules/swarm/SConstruct index a7201cb..089305e 100644 --- a/examples/cavity_2d/SConstruct +++ b/modules/swarm/SConstruct @@ -32,7 +32,6 @@ def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, ta def isAbsolutePath(key, dirname, env): assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) - env_vars = Variables( args=ARGUMENTS ) @@ -43,11 +42,6 @@ env_vars.Add('prefix', validator=isAbsolutePath ) -env_vars.Add('build_examples', - help='If examples should be built', - default="true" -) - env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], CPPDEFINES=['SAW_UNIX'], CXXFLAGS=[ @@ -70,10 +64,12 @@ env.headers = []; env.targets = []; Export('env') -SConscript('SConscript') +SConscript('c++/SConscript') +SConscript('tests/SConscript') env.Alias('cdb', env.cdb); env.Alias('all', [env.targets]); env.Default('all'); + env.Alias('install', '$prefix') diff --git a/modules/swarm/c++/SConscript b/modules/swarm/c++/SConscript new file mode 100644 index 0000000..4071add --- /dev/null +++ b/modules/swarm/c++/SConscript @@ -0,0 +1,27 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +swarm_env = env.Clone(); + +swarm_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +swarm_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += swarm_env.sources; +env.headers += swarm_env.headers; + +# Header only, like core. The StaticLibrary exists so dependents have something +# to name; it is built from zero translation units. +objects = [] +swarm_env.add_source_files(objects, swarm_env.sources, shared=False); +env.library_static = swarm_env.StaticLibrary('#build/kel-lbm-swarm', [objects]); + +env.Install('$prefix/lib/', env.library_static); +env.Install('$prefix/include/kel/lbm/swarm/', swarm_env.headers); diff --git a/modules/swarm/c++/capability.hpp b/modules/swarm/c++/capability.hpp new file mode 100644 index 0000000..8497969 --- /dev/null +++ b/modules/swarm/c++/capability.hpp @@ -0,0 +1,199 @@ +#pragma once + +#include <kel/lbm/abstract/schema.hpp> + +/** + * What a worker tells the orchestrator about itself. + * + * The envelope is deliberately backend agnostic: common fields, then one + * section named by "kind". Browsers running WebGL2 and native processes are + * both first class worker kinds, and the swarm is meant to take WebGPU runners + * later; a string tag survives version skew where a positional format does not. + * + * A worker of another kind carries its own section instead. Nothing has to + * carry a section it has no business describing, because each kind gets its own + * full capability type -- see CapabilityOf below and the hello variants in + * message.hpp. + */ +namespace kel { +namespace lbm { +namespace sch { + +/** + * Queryable WebGL2 limits, straight from glGetIntegerv. + * + * Worth knowing which of these actually bind. max_draw_buffers and + * max_color_attachments are guaranteed to be only 4, and since a D2Q9 lattice + * needs ceil(9/4) = 3 render targets they are the limits that decide whether a + * descriptor fits in a single pass at all. The rest are either generous in + * practice or bound by memory long before they bind by count. + */ +using GlLimits = Struct< + Member<UInt32, "max_texture_size">, + Member<UInt32, "max_3d_texture_size">, + Member<UInt32, "max_array_texture_layers">, + Member<UInt32, "max_draw_buffers">, + Member<UInt32, "max_color_attachments">, + Member<UInt32, "max_texture_image_units">, + Member<UInt32, "max_combined_texture_image_units">, + Member<UInt32, "max_renderbuffer_size">, + Member<UInt32, "max_fragment_uniform_vectors">, + Member<FixedArray<UInt32, 2u>, "max_viewport_dims"> +>; + +/** + * Extension and context properties. + * + * color_buffer_float is a hard gate: without EXT_color_buffer_float a float + * texture is not color renderable and there is no f32 solver on this device at + * all. The others are reported so the orchestrator can decide, not so the + * worker can. + */ +using GlFeatures = Struct< + Member<Bool, "webgl2">, + Member<Bool, "offscreen_canvas">, + Member<Bool, "color_buffer_float">, + Member<Bool, "color_buffer_half_float">, + Member<Bool, "texture_float_linear">, + Member<Bool, "float_blend">, + Member<Bool, "parallel_shader_compile">, + Member<Bool, "disjoint_timer_query"> +>; + +/** + * What the worker measured, as opposed to what it was told. + * + * There is no queryable "available VRAM" in WebGL2 -- navigator.deviceMemory is + * system RAM and unrelated -- and max_texture_size overstates capacity by + * roughly a hundredfold on a typical desktop. So these come from an actual + * allocation trial. + * + * probed_bytes_max is the largest allocation that survived; probed_bytes_safe + * applies safety_factor to leave room for the canvas, the compositor and other + * tabs. Both are reported so the orchestrator can apply its own policy rather + * than inheriting the worker's. + * + * probe_hit_texture_size_bound distinguishes "ran out of memory" from "ran into + * max_texture_size", which are very different signals about a device. + */ +using GlBudget = Struct< + Member<UInt64, "probed_bytes_max">, + Member<UInt64, "probed_bytes_safe">, + Member<UInt32, "probe_trials">, + Member<Bool, "context_lost_during_probe">, + Member<Bool, "probe_hit_texture_size_bound">, + Member<Float32, "safety_factor"> +>; + +/** + * One feasible way to lay a lattice out in textures on this device. + * + * This type has three jobs and keeping it as one type is what stops them + * drifting: it is an element of the capability report, it is the assignment + * message the orchestrator sends back, and it is the input to the shader + * preamble generator. + * + * The slot tables are the indirection that makes pixel stuffing expressible -- + * slot_tex_of_q[i] / slot_cmp_of_q[i] say which texture and which channel + * direction i lives in, and the aux tables do the same for the non + * distribution fields packed into otherwise idle slots. + */ +using PackingPlan = Struct< + Member<String, "layout">, + Member<String, "aux_mode">, + Member<UInt8, "d">, + Member<UInt8, "q">, + Member<UInt8, "ghost">, + Member<UInt8, "storage_bits">, + Member<UInt8, "slots_per_texel">, + Member<UInt8, "textures_df">, + Member<UInt8, "textures_aux">, + Member<UInt8, "states">, + Member<UInt8, "passes">, + Member<UInt8, "idle_slots">, + Member<FixedArray<UInt32, 3u>, "domain">, + Member<FixedArray<UInt32, 2u>, "tiles">, + Member<Bool, "tiles_pow2">, + Member<FixedArray<UInt32, 2u>, "texture_size">, + Member<UInt64, "bytes_total">, + Member<UInt64, "bytes_useful">, + Member<Array<UInt8, 1u>, "slot_tex_of_q">, + Member<Array<UInt8, 1u>, "slot_cmp_of_q">, + Member<Array<UInt8, 1u>, "slot_tex_of_aux">, + Member<Array<UInt8, 1u>, "slot_cmp_of_aux">, + Member<Array<String, 1u>, "aux_names"> +>; + +using WebGl2Capability = Struct< + Member<GlLimits, "limits">, + Member<GlFeatures, "features">, + Member<GlBudget, "budget">, + Member<Array<PackingPlan, 1u>, "feasible"> +>; + +/** + * A worker that is a process rather than a browser tab. + * + * Native workers are not only a test fixture -- they are a worker kind in their + * own right, and the orchestrator should be able to hand them real work. What + * they report is correspondingly different: there is no texture budget to probe + * because there are no textures, so the interesting numbers are threads and + * addressable memory. + */ +using NativeCapability = Struct< + Member<UInt32, "threads">, + Member<UInt64, "memory_bytes">, + Member<String, "cpu">, + Member<Bool, "has_sycl"> +>; + +/** + * The part every worker reports, whatever it is. + * + * This is deliberately the *whole* type rather than a prefix of a larger one. + * Decoding runs strict, so a struct naming every backend section would oblige a + * browser to describe a CPU and a native process to describe a texture budget. + * Instead the common part stands alone and is enough to route on: decode this, + * read "kind", then decode again as the matching full type below. The backend + * section is skipped on the first pass because unknown keys always are. + */ +using WorkerCapability = Struct< + Member<String, "worker_id">, + Member<String, "kind">, + Member<String, "schema_version">, + Member<UInt64, "reported_at_ms">, + Member<String, "user_agent"> +>; + +/** + * The common part plus exactly one backend section. One alias per kind, so + * adding a kind never widens what the existing ones have to fill in. + */ +template <typename Section, string_literal Key> +using CapabilityOf = Struct< + Member<String, "worker_id">, + Member<String, "kind">, + Member<String, "schema_version">, + Member<UInt64, "reported_at_ms">, + Member<String, "user_agent">, + Member<Section, Key> +>; + +using WebGl2WorkerCapability = CapabilityOf<WebGl2Capability, "webgl2">; +using NativeWorkerCapability = CapabilityOf<NativeCapability, "native">; + +/** + * The small, frequently resent part: enough to monitor a worker without + * re-sending its capability. Rides along on pong so a heartbeat doubles as a + * status update. + */ +using WorkerLive = Struct< + Member<String, "state">, + Member<UInt64, "uptime_ms">, + Member<UInt64, "steps_done">, + Member<String, "last_error"> +>; + +} // namespace sch +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/capacity.hpp b/modules/swarm/c++/capacity.hpp new file mode 100644 index 0000000..4073405 --- /dev/null +++ b/modules/swarm/c++/capacity.hpp @@ -0,0 +1,236 @@ +#pragma once + +#include <cstdint> + +/** + * How much lattice fits on a worker. + * + * This is pure arithmetic over a descriptor's Q and a device's reported limits, + * with no GL and no schema, which is the point: the orchestrator needs it to + * size assignments, the worker needs it to drive its allocation trial, and + * neither reason is a good one to make it untestable without a browser. + * + * Everything is constexpr so the ceilings can be asserted at compile time + * against a descriptor rather than discovered on a device that happens to be + * minimum spec. + */ +namespace kel { +namespace lbm { +namespace swarm { + +/** + * A WebGL2 implementation is only *guaranteed* these. Real desktop values are + * far higher, but a plan that relies on more than this is a plan that works on + * the developer's machine. + */ +struct webgl2_guaranteed { + static constexpr uint32_t max_texture_size = 2048u; + static constexpr uint32_t max_3d_texture_size = 256u; + static constexpr uint32_t max_array_texture_layers = 256u; + static constexpr uint32_t max_draw_buffers = 4u; + static constexpr uint32_t max_color_attachments = 4u; + static constexpr uint32_t max_texture_image_units = 16u; +}; + +/** + * Packing a Q-direction lattice into RGBA textures. + * + * One texel holds four scalars, so a descriptor needs ceil(Q/4) textures and + * wastes whatever is left over. The leftovers are not merely waste: the aux + * fields (the per cell info flag, later the particle fields) can be stuffed + * into them, which is why idle_slots is reported rather than just computed. + */ +struct packing_math { + static constexpr uint64_t slots_per_texel = 4u; + + static constexpr uint64_t planes(uint64_t q){ + return (q + slots_per_texel - 1u) / slots_per_texel; + } + + static constexpr uint64_t idle_slots(uint64_t q){ + return planes(q) * slots_per_texel - q; + } + + /** + * Aux planes needed once the idle slots have been used. Stuffing only + * works when the leftovers are big enough; D2Q9 has three idle slots, so + * a single info flag fits but the four fields a PSM run wants do not. + */ + static constexpr uint64_t aux_planes(uint64_t q, uint64_t aux_scalars, + bool stuffed){ + if(stuffed and aux_scalars <= idle_slots(q)){ + return 0u; + } + return (aux_scalars + slots_per_texel - 1u) / slots_per_texel; + } + + static constexpr uint64_t bytes_per_texel(uint64_t storage_bits){ + return slots_per_texel * storage_bits / 8u; + } + + /** + * Distributions are double buffered for the ping pong; aux fields are not, + * since nothing writes them during a step. + */ + static constexpr uint64_t bytes_per_cell(uint64_t q, uint64_t states, + uint64_t aux_scalars, bool stuffed, + uint64_t storage_bits = 32u){ + const uint64_t tex = states * planes(q) + aux_planes(q, aux_scalars, stuffed); + return tex * bytes_per_texel(storage_bits); + } + + /** + * Render targets needed per pass, against what the device allows. + * + * This is the limit that actually decides whether a descriptor is usable. + * Collision couples every direction, so a multi-pass split has to re-read + * all planes and recompute rho and u in each pass -- roughly double the + * arithmetic for the same bandwidth. Worth detecting rather than + * stumbling into. + */ + static constexpr uint64_t passes(uint64_t q, uint64_t mrt){ + if(mrt == 0u){ + return 0u; + } + return (planes(q) + mrt - 1u) / mrt; + } + + static constexpr bool fits_single_pass(uint64_t q, uint64_t mrt){ + return passes(q, mrt) == 1u; + } +}; + +/** + * Integer square root by bisection. + * + * std::sqrt is not constexpr before C++26 and this has to run at compile time + * for the ceiling assertions. Exactness also matters more than speed here: a + * domain edge that rounds up by one costs a whole row of cells against a + * budget that was measured, not guessed. + */ +constexpr uint64_t isqrt(uint64_t value){ + if(value < 2u){ + return value; + } + + uint64_t low = 1u; + uint64_t high = (value / 2u) + 1u; + + while(low < high){ + const uint64_t mid = low + (high - low + 1u) / 2u; + if(mid <= value / mid){ + low = mid; + } else { + high = mid - 1u; + } + } + + return low; +} + +/** + * A 2D domain and what it costs. + */ +struct domain_2d { + uint64_t nx; + uint64_t ny; + + constexpr uint64_t cells() const { return nx * ny; } +}; + +struct capacity_math { + /** + * Cells the texture dimension allows, independent of memory. + * + * On a typical desktop this is around a hundred times looser than what + * memory actually permits, which is exactly why the worker measures + * instead of trusting it -- but it is still a hard ceiling and a worker + * that hits *this* one is telling you something different about itself + * than a worker that ran out of memory. + */ + static constexpr uint64_t texture_bound_cells_2d(uint64_t max_texture_size, + uint64_t ghost){ + const uint64_t pad = 2u * ghost; + if(max_texture_size <= pad){ + return 0u; + } + const uint64_t edge = max_texture_size - pad; + return edge * edge; + } + + static constexpr uint64_t cells_from_bytes(uint64_t bytes, + uint64_t bytes_per_cell){ + if(bytes_per_cell == 0u){ + return 0u; + } + return bytes / bytes_per_cell; + } + + static constexpr uint64_t max_cells_2d(uint64_t budget_bytes, + uint64_t bytes_per_cell, + uint64_t max_texture_size, + uint64_t ghost){ + const uint64_t by_memory = cells_from_bytes(budget_bytes, bytes_per_cell); + const uint64_t by_texture = texture_bound_cells_2d(max_texture_size, ghost); + return (by_memory < by_texture) ? by_memory : by_texture; + } + + /** + * Largest domain of a given aspect ratio that fits in a cell count. + * + * Aspect arrives as a ratio of integers rather than a float so the result + * is reproducible across the orchestrator and the worker; they must agree + * exactly or an assignment will not fit the allocation the worker made. + * + * Rounds down on both axes, and clamps each edge to what the texture + * dimension allows, so the answer is always feasible rather than merely + * close. + */ + static constexpr domain_2d fit_domain_2d(uint64_t cells, + uint64_t aspect_num, + uint64_t aspect_den, + uint64_t max_texture_size, + uint64_t ghost){ + if(cells == 0u or aspect_num == 0u or aspect_den == 0u){ + return domain_2d{0u, 0u}; + } + + // ny = sqrt(cells * den / num), nx = ny * num / den + uint64_t ny = isqrt((cells * aspect_den) / aspect_num); + if(ny == 0u){ + ny = 1u; + } + uint64_t nx = (ny * aspect_num) / aspect_den; + if(nx == 0u){ + nx = 1u; + } + + const uint64_t pad = 2u * ghost; + const uint64_t edge = (max_texture_size > pad) ? (max_texture_size - pad) : 0u; + if(edge == 0u){ + return domain_2d{0u, 0u}; + } + if(nx > edge){ + nx = edge; + } + if(ny > edge){ + ny = edge; + } + + // Rounding up through the aspect ratio can overshoot the budget by a + // row; give the row back rather than hand out a domain that does not + // fit the measurement. + while(nx * ny > cells and ny > 1u){ + --ny; + } + while(nx * ny > cells and nx > 1u){ + --nx; + } + + return domain_2d{nx, ny}; + } +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/interpret.hpp b/modules/swarm/c++/interpret.hpp new file mode 100644 index 0000000..1b2dcfa --- /dev/null +++ b/modules/swarm/c++/interpret.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include "capability.hpp" +#include "capacity.hpp" + +#include <kel/lbm/abstract/data.hpp> + +#include <cstdint> +#include <iomanip> +#include <sstream> +#include <string> + +/** + * Turning a reported capability into the one number the orchestrator needs: + * how much lattice this worker can hold. + * + * Quoted in D2Q9 cells so worker kinds are directly comparable. A browser's + * figure comes from its measured texture budget; a native worker's from + * addressable memory. Neither is a promise -- both ignore everything else + * happening on the machine -- but a planning figure that means the same thing + * across kinds is what makes a heterogeneous registry useful rather than + * decorative. + * + * Here rather than in the orchestrator because it is pure and therefore worth + * testing, and because the browser worker may eventually want to reason about + * its own capacity the same way the orchestrator does. + */ +namespace kel { +namespace lbm { +namespace swarm { + +/** + * The reference packing every capacity figure is quoted in: D2Q9, double + * buffered, the info flag stuffed into an idle slot. 96 bytes per cell. + */ +constexpr uint64_t reference_bytes_per_cell(){ + return packing_math::bytes_per_cell(9u, 2u, 1u, true); +} + +namespace impl { + +inline std::string human_bytes(uint64_t bytes){ + std::ostringstream oss; + oss << std::fixed << std::setprecision(1); + + if(bytes >= (1ull << 30)){ + oss << (static_cast<double>(bytes) / static_cast<double>(1ull << 30)) + << " GiB"; + } else if(bytes >= (1ull << 20)){ + oss << (static_cast<double>(bytes) / static_cast<double>(1ull << 20)) + << " MiB"; + } else { + oss << bytes << " B"; + } + + return oss.str(); +} + +} // namespace impl + +inline uint64_t webgl2_capacity_cells(const data<sch::WebGl2Capability> &cap){ + const uint64_t budget = cap.get<"budget">().get<"probed_bytes_safe">().get(); + const uint64_t max_tex = cap.get<"limits">().get<"max_texture_size">().get(); + + // Both bounds, tighter wins -- a worker can be limited by either, and + // which one it is says something different about the device. + return capacity_math::max_cells_2d(budget, reference_bytes_per_cell(), + max_tex, 0u); +} + +inline uint64_t native_capacity_cells(const data<sch::NativeCapability> &cap){ + const uint64_t memory = cap.get<"memory_bytes">().get(); + + // No texture dimension to bind against, so memory is the only limit. Take + // a quarter of it: the rest of the machine exists. + return capacity_math::cells_from_bytes(memory / 4u, + reference_bytes_per_cell()); +} + +inline std::string webgl2_summary(const data<sch::WebGl2Capability> &cap){ + std::ostringstream oss; + + oss << "tex " << cap.get<"limits">().get<"max_texture_size">().get() + << ", mrt " << cap.get<"limits">().get<"max_draw_buffers">().get() + << ", budget " << impl::human_bytes( + cap.get<"budget">().get<"probed_bytes_safe">().get()); + + if(not cap.get<"features">().get<"color_buffer_float">().get()){ + // Without this there is no float render target and so no solver at + // all. Worth shouting about rather than leaving to be inferred. + oss << " [NO FLOAT RENDER]"; + } + + return oss.str(); +} + +inline std::string native_summary(const data<sch::NativeCapability> &cap){ + std::ostringstream oss; + + oss << cap.get<"threads">().get() << " threads, " + << impl::human_bytes(cap.get<"memory_bytes">().get()) + << ", " << cap.get<"cpu">().stl_string(); + + if(cap.get<"has_sycl">().get()){ + oss << ", sycl"; + } + + return oss.str(); +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/message.hpp b/modules/swarm/c++/message.hpp new file mode 100644 index 0000000..aba5a9c --- /dev/null +++ b/modules/swarm/c++/message.hpp @@ -0,0 +1,140 @@ +#pragma once + +#include "capability.hpp" + +#include <string_view> + +/** + * The worker/orchestrator control protocol. + * + * Every message is a JSON object carrying the same three leading members -- + * "type", "worker_id", "seq" -- followed by whatever that type adds. Decoding + * happens in two stages (see wire.hpp): read MsgHeader to learn the type, then + * decode the full typed struct. + * + * The reason the protocol is shaped this way rather than as a tagged union is + * that sch::Union has no JSON codec. It turns out not to matter: the Struct + * decoder skips keys it does not recognise, so reading a header out of a full + * message is well defined and costs one extra parse of a short string. + * + * A versioning note that the strictness buys and charges for. Wire codecs run + * strict, so a *missing* member is an error while an *unknown* member is + * ignored. That means a newer sender talking to an older receiver is fine, but + * an older sender talking to a newer receiver fails as soon as the newer side + * adds a member. Adding a field to a message is therefore a breaking change in + * one direction, which is what schema_version in the hello/welcome exchange is + * for -- catch the mismatch deliberately instead of discovering it as a + * confusing decode error. + */ +namespace kel { +namespace lbm { + +namespace sch { + +/** + * Prepends the common members so each message type only states what it adds. + */ +template <typename... Members> +using Envelope = Struct< + Member<String, "type">, + Member<String, "worker_id">, + Member<UInt64, "seq">, + Members... +>; + +/// Just the common members. Decoded first to learn which type follows. +using MsgHeader = Envelope<>; + +/** + * worker -> orchestrator. First message on the channel. + * + * MsgHello carries only the common capability fields, which is enough to learn + * the worker kind. Decode it first, read capability.kind, then decode the same + * text again as the matching variant below -- the backend section is skipped on + * the first pass because unknown keys always are. + * + * The alternative, one hello naming every backend section, does not work: wire + * codecs run strict, so it would oblige a browser to describe a CPU. + */ +using MsgHello = Envelope<Member<WorkerCapability, "capability">>; + +using MsgHelloWebGl2 = Envelope<Member<WebGl2WorkerCapability, "capability">>; +using MsgHelloNative = Envelope<Member<NativeWorkerCapability, "capability">>; + +/// orchestrator -> worker. Confirms registration and sets the heartbeat pace. +using MsgWelcome = Envelope< + Member<UInt64, "heartbeat_ms">, + Member<String, "schema_version"> +>; + +/// orchestrator -> worker. +using MsgPing = Envelope<Member<UInt64, "sent_at_ms">>; + +/** + * worker -> orchestrator. Echoes sent_at_ms so round trip time is measured + * against the orchestrator's own clock and the two machines never have to agree + * on what time it is. + */ +using MsgPong = Envelope< + Member<UInt64, "sent_at_ms">, + Member<UInt64, "worker_now_ms">, + Member<WorkerLive, "live"> +>; + +/// worker -> orchestrator. Initialised and idle. +using MsgReady = Envelope<Member<WorkerLive, "live">>; + +/// worker -> orchestrator. Unsolicited: a state change or an error. +using MsgStatus = Envelope<Member<WorkerLive, "live">>; + +} // namespace sch + +/** + * Wire values for the "type" member. + * + * Deliberately strings rather than an integer enum: the orchestrator and the + * worker are built from the same source today but will not always be, and a + * string tag misread is a routing failure you can see in a log, where an + * integer tag misread is a silently wrong payload. + */ +namespace msg { +inline constexpr std::string_view hello = "hello"; +inline constexpr std::string_view welcome = "welcome"; +inline constexpr std::string_view ping = "ping"; +inline constexpr std::string_view pong = "pong"; +inline constexpr std::string_view ready = "ready"; +inline constexpr std::string_view status = "status"; + +/// orchestrator -> everyone. A run begins or ends. +inline constexpr std::string_view run_start = "run_start"; +inline constexpr std::string_view run_end = "run_end"; +} // namespace msg + +/** + * Values for WorkerLive's "state". The orchestrator owns stale and gone -- a + * worker never reports those about itself, they are conclusions drawn from + * missed heartbeats. + */ +namespace wstate { +inline constexpr std::string_view registered = "registered"; +inline constexpr std::string_view ready = "ready"; +inline constexpr std::string_view busy = "busy"; +inline constexpr std::string_view error = "error"; +inline constexpr std::string_view stale = "stale"; +inline constexpr std::string_view gone = "gone"; +} // namespace wstate + +/** + * Bumped whenever a message or capability schema changes shape. Compared on + * hello/welcome so a mismatch is reported rather than hit as a decode failure. + */ +inline constexpr std::string_view schema_version = "1"; + +/// Worker kinds, the tag selecting which capability section is populated. +namespace wkind { +inline constexpr std::string_view webgl2 = "webgl2"; +inline constexpr std::string_view native = "native"; +} // namespace wkind + +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/probe_search.hpp b/modules/swarm/c++/probe_search.hpp new file mode 100644 index 0000000..f678937 --- /dev/null +++ b/modules/swarm/c++/probe_search.hpp @@ -0,0 +1,198 @@ +#pragma once + +#include <cstdint> + +/** + * Finding out how much a device will actually give you. + * + * WebGL2 reports limits but not capacity. MAX_TEXTURE_SIZE on a typical desktop + * implies something like 25 GB of lattice, roughly a hundred times what the GPU + * will really hand over, and there is no query for the real figure -- + * navigator.deviceMemory is system RAM and unrelated. So capacity is measured: + * try an allocation, see whether it survives, adjust. + * + * The strategy lives here rather than next to the GL calls because it is + * ordinary arithmetic and deserves to be tested as such. The caller owns the + * part that cannot be tested without a browser -- actually allocating -- and + * feeds the answers back in: + * + * allocation_search search{cfg}; + * while(not search.done()){ + * search.report(try_allocate(search.candidate())); + * } + * use(search.best_cells()); + * + * Grow by doubling until something fails, then bisect. Doubling finds the right + * order of magnitude in a handful of steps even when devices differ by a factor + * of a thousand; bisecting then tightens it without the linear crawl a pure + * step search would need. + */ +namespace kel { +namespace lbm { +namespace swarm { + +class allocation_search { +public: + struct config { + /** + * First attempt. Deliberately small: the opening trial should succeed + * on anything that can run at all, because a failure here means the + * device is unusable rather than merely modest. + */ + uint64_t start_cells = 65536u; + + /** + * Hard ceiling from the texture dimension, in cells. Zero means no + * ceiling. Reaching it is a different answer from running out of + * memory and is reported separately. + */ + uint64_t ceiling_cells = 0u; + + /// Bisection steps once a failure has bracketed the answer. Four gets + /// within about a sixteenth of the bracket, which is finer than the + /// figure deserves given it is a moving target anyway. + uint64_t bisect_steps = 4u; + + /// Backstop so a device that answers inconsistently cannot spin. + uint64_t max_trials = 32u; + }; + + explicit allocation_search(config cfg) : cfg_{cfg} { + candidate_ = cfg_.start_cells; + + if(cfg_.ceiling_cells > 0u and candidate_ > cfg_.ceiling_cells){ + candidate_ = cfg_.ceiling_cells; + } + + if(candidate_ == 0u){ + phase_ = phase::finished; + } + } + + bool done() const { return phase_ == phase::finished; } + + /// The cell count to attempt next. Meaningless once done(). + uint64_t candidate() const { return candidate_; } + + /// Largest allocation that survived. Zero means nothing did. + uint64_t best_cells() const { return best_; } + + uint64_t trials() const { return trials_; } + + /** + * True when the search stopped because the texture dimension ran out + * rather than because memory did. Worth distinguishing: one says the + * device is small, the other says it is oddly shaped. + */ + bool hit_ceiling() const { return hit_ceiling_; } + + void report(bool ok){ + if(phase_ == phase::finished){ + return; + } + + ++trials_; + + if(phase_ == phase::grow){ + report_grow(ok); + } else { + report_bisect(ok); + } + + if(trials_ >= cfg_.max_trials){ + phase_ = phase::finished; + } + } + +private: + enum class phase { grow, bisect, finished }; + + void report_grow(bool ok){ + if(not ok){ + failed_at_ = candidate_; + + if(best_ == 0u){ + // Even the opening attempt failed. There is nothing to + // bisect between, and the honest answer is zero. + phase_ = phase::finished; + return; + } + + phase_ = phase::bisect; + advance_bisect(); + return; + } + + best_ = candidate_; + + if(cfg_.ceiling_cells > 0u and candidate_ >= cfg_.ceiling_cells){ + // The texture dimension, not memory, is what stopped us. + hit_ceiling_ = true; + phase_ = phase::finished; + return; + } + + uint64_t next = candidate_ * 2u; + if(next < candidate_){ + // Overflow. Nothing sane gets here, but silently wrapping to a + // tiny candidate would turn a huge device into a small one. + phase_ = phase::finished; + return; + } + if(cfg_.ceiling_cells > 0u and next > cfg_.ceiling_cells){ + next = cfg_.ceiling_cells; + } + if(next == candidate_){ + phase_ = phase::finished; + return; + } + + candidate_ = next; + } + + void report_bisect(bool ok){ + if(ok){ + best_ = candidate_; + } else { + failed_at_ = candidate_; + } + advance_bisect(); + } + + void advance_bisect(){ + if(bisects_done_ >= cfg_.bisect_steps){ + phase_ = phase::finished; + return; + } + if(failed_at_ <= best_ + 1u){ + // Bracket closed; there is nothing left between them. + phase_ = phase::finished; + return; + } + + const uint64_t mid = best_ + ((failed_at_ - best_) / 2u); + if(mid == best_ or mid == failed_at_){ + phase_ = phase::finished; + return; + } + + candidate_ = mid; + ++bisects_done_; + } + + config cfg_; + + phase phase_ = phase::grow; + uint64_t candidate_ = 0u; + /// Largest known good. + uint64_t best_ = 0u; + /// Smallest known bad. + uint64_t failed_at_ = 0u; + uint64_t trials_ = 0u; + uint64_t bisects_done_ = 0u; + bool hit_ceiling_ = false; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/registry.hpp b/modules/swarm/c++/registry.hpp new file mode 100644 index 0000000..6159a72 --- /dev/null +++ b/modules/swarm/c++/registry.hpp @@ -0,0 +1,427 @@ +#pragma once + +#include "capability.hpp" + +#include <kel/lbm/abstract/data.hpp> + +#include <cstdint> +#include <iomanip> +#include <map> +#include <mutex> +#include <optional> +#include <sstream> +#include <string> +#include <vector> + +/** + * Who is connected, how healthy they are, and how much they can hold. + * + * Deliberately free of any transport. The registry never sees a socket, a peer + * connection or a data channel -- it is given a session id and timestamps, and + * it answers with decisions. Two reasons: + * + * 1. modules/swarm is compiled by em++ as well as by the host compiler, and + * libdatachannel does not exist in a browser. Anything in here that + * touched it would make the worker unbuildable. + * 2. The liveness state machine is the part most worth testing and the part + * least convenient to test through a real connection. Every entry point + * takes `now_ms` explicitly rather than reading a clock, so a test can + * drive a worker from healthy to dropped in a few microseconds and assert + * each transition on the way. + * + * Resolving a session id back to something sendable is the caller's job. + */ +namespace kel { +namespace lbm { +namespace swarm { + +enum class worker_state { + /// Connected, peer connection up, but no hello yet. + signalling, + /// Hello received and accepted. + registered, + /// Worker says it is initialised and idle. + ready, + /// Missed enough heartbeats to be doubtful, not enough to give up. + stale, +}; + +inline const char *to_string(worker_state state){ + switch(state){ + case worker_state::signalling: return "signalling"; + case worker_state::registered: return "registered"; + case worker_state::ready: return "ready"; + case worker_state::stale: return "stale"; + } + return "?"; +} + +/** + * Liveness thresholds. + * + * At namespace scope rather than nested in registry so it is a complete type + * where the constructor default argument needs it. + */ +struct worker_thresholds { + /// Missed pings before a worker is called stale. + uint64_t stale_after = 3u; + /// Missed pings before it is dropped entirely. + uint64_t drop_after = 6u; +}; + +/** + * What the orchestrator knows about one worker. + * + * Note which id is which. session_id is assigned by the orchestrator and is + * unique by construction; worker_id is whatever the worker called itself and is + * neither unique nor trustworthy. The registry is keyed on the former and + * merely carries the latter, so two workers choosing the same name is a display + * annoyance rather than a collision. + */ +struct worker_record { + std::string session_id; + std::string worker_id; + std::string kind; + + worker_state state = worker_state::signalling; + + uint64_t connected_at_ms = 0u; + uint64_t registered_at_ms = 0u; + uint64_t last_seen_ms = 0u; + + /// Sequence of the most recent ping, so a late pong can be told from a + /// current one rather than silently resetting the miss counter. + uint64_t ping_seq = 0u; + uint64_t outstanding_pings = 0u; + uint64_t pongs = 0u; + + /// Exponentially weighted mean round trip, milliseconds. Negative until + /// the first pong lands. + double rtt_ewma_ms = -1.0; + + /** + * Roughly how many lattice cells this worker could hold, derived from what + * it reported. The whole point of a capability is to answer this, so it is + * computed once on registration rather than recomputed per query. + */ + uint64_t capacity_cells = 0u; + + /// Short human readable form of the capability, for the monitor. + std::string summary; + + /// Last live block the worker sent, on a pong or a status. + data<sch::WorkerLive> live; +}; + +/** + * The orchestrator's only shared mutable state. + * + * libdatachannel delivers callbacks on its own threads and the heartbeat runs + * on another, so everything here is behind one mutex. The interface is + * deliberately coarse -- whole operations rather than accessors -- so callers + * cannot hold a reference to a record past the lock, which is the usual way a + * design like this goes wrong. + */ +class registry { +public: + using thresholds = worker_thresholds; + + explicit registry(worker_thresholds limits = {}) : limits_{limits} {} + + void add_session(const std::string &session_id, uint64_t now_ms){ + std::lock_guard<std::mutex> lock{mutex_}; + + worker_record rec; + rec.session_id = session_id; + rec.state = worker_state::signalling; + rec.connected_at_ms = now_ms; + rec.last_seen_ms = now_ms; + + records_[session_id] = std::move(rec); + } + + /// Returns false if the session is gone. + bool mark_registered(const std::string &session_id, + const std::string &worker_id, const std::string &kind, + uint64_t capacity_cells, std::string summary, + uint64_t now_ms){ + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = records_.find(session_id); + if(iter == records_.end()){ + return false; + } + + auto &rec = iter->second; + rec.worker_id = worker_id; + rec.kind = kind; + rec.capacity_cells = capacity_cells; + rec.summary = std::move(summary); + rec.state = worker_state::registered; + rec.registered_at_ms = now_ms; + rec.last_seen_ms = now_ms; + + return true; + } + + bool mark_ready(const std::string &session_id, data<sch::WorkerLive> live, + uint64_t now_ms){ + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = records_.find(session_id); + if(iter == records_.end()){ + return false; + } + + iter->second.state = worker_state::ready; + iter->second.live = std::move(live); + iter->second.last_seen_ms = now_ms; + + return true; + } + + bool update_live(const std::string &session_id, data<sch::WorkerLive> live, + uint64_t now_ms){ + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = records_.find(session_id); + if(iter == records_.end()){ + return false; + } + + iter->second.live = std::move(live); + iter->second.last_seen_ms = now_ms; + + return true; + } + + /** + * Records a pong. `echoed_sent_at_ms` is the orchestrator's own timestamp + * coming back, so round trip needs no agreement between the two clocks. + * Ignores the liveness effect of a pong whose sequence is not current. + */ + bool record_pong(const std::string &session_id, uint64_t seq, + uint64_t echoed_sent_at_ms, data<sch::WorkerLive> live, + uint64_t now_ms){ + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = records_.find(session_id); + if(iter == records_.end()){ + return false; + } + + auto &rec = iter->second; + + if(seq != rec.ping_seq){ + // A pong for a ping we have already given up on. Counting it would + // paper over a worker that is consistently one beat behind, so the + // liveness side is left alone; the message still proves the channel + // works, which is worth recording. + rec.last_seen_ms = now_ms; + return true; + } + + rec.outstanding_pings = 0u; + ++rec.pongs; + rec.last_seen_ms = now_ms; + rec.live = std::move(live); + + // Guard against a clock that went backwards rather than producing a + // negative round trip. + const double rtt = (now_ms > echoed_sent_at_ms) + ? static_cast<double>(now_ms - echoed_sent_at_ms) + : 0.0; + + if(rec.rtt_ewma_ms < 0.0){ + rec.rtt_ewma_ms = rtt; + } else { + // Smoothing factor chosen to react within a few beats without + // letting one slow sample dominate. + constexpr double alpha = 0.25; + rec.rtt_ewma_ms = (alpha * rtt) + ((1.0 - alpha) * rec.rtt_ewma_ms); + } + + // A worker that answers again stops being doubtful. It goes back to + // registered rather than ready, because readiness is something only the + // worker can assert. + if(rec.state == worker_state::stale){ + rec.state = worker_state::registered; + } + + return true; + } + + /// Removes a session. Returns the record if there was one. + std::optional<worker_record> remove(const std::string &session_id){ + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = records_.find(session_id); + if(iter == records_.end()){ + return std::nullopt; + } + + worker_record rec = std::move(iter->second); + records_.erase(iter); + + return rec; + } + + /** + * One heartbeat tick. + * + * Bumps the ping sequence and the miss counter for every session, applies + * the stale and drop thresholds, and returns what to do. Nothing is sent + * from here -- the registry has nothing to send on, and in the orchestrator + * sending runs callbacks, which while holding this lock is how it would + * deadlock. + */ + struct tick_result { + struct ping_target { + std::string session_id; + uint64_t seq; + }; + + std::vector<ping_target> to_ping; + /// Sessions that passed drop_after; already removed from the registry. + std::vector<worker_record> dropped; + /// Sessions that just crossed into stale, for logging. + std::vector<std::string> newly_stale; + }; + + tick_result tick(uint64_t now_ms){ + (void) now_ms; + + tick_result out; + + std::lock_guard<std::mutex> lock{mutex_}; + + for(auto iter = records_.begin(); iter != records_.end(); ){ + auto &rec = iter->second; + + // A session still negotiating has no channel to ping on. It is not + // unhealthy, it is just not up yet, so the thresholds do not apply. + if(rec.state == worker_state::signalling){ + ++iter; + continue; + } + + if(rec.outstanding_pings >= limits_.drop_after){ + out.dropped.push_back(rec); + iter = records_.erase(iter); + continue; + } + + if(rec.outstanding_pings >= limits_.stale_after and + rec.state != worker_state::stale){ + rec.state = worker_state::stale; + out.newly_stale.push_back(rec.session_id); + } + + ++rec.ping_seq; + ++rec.outstanding_pings; + + out.to_ping.push_back( + tick_result::ping_target{rec.session_id, rec.ping_seq}); + + ++iter; + } + + return out; + } + + std::size_t size() const { + std::lock_guard<std::mutex> lock{mutex_}; + return records_.size(); + } + + /// Snapshot for display. Copies, so the caller holds no lock. + std::vector<worker_record> snapshot() const { + std::lock_guard<std::mutex> lock{mutex_}; + + std::vector<worker_record> out; + out.reserve(records_.size()); + for(const auto &entry : records_){ + out.push_back(entry.second); + } + + return out; + } + +private: + mutable std::mutex mutex_; + std::map<std::string, worker_record> records_; + worker_thresholds limits_; +}; + +namespace impl { + +inline std::string human_cells(uint64_t cells){ + std::ostringstream oss; + oss << std::fixed << std::setprecision(1); + + if(cells >= 1000000000ull){ + oss << (static_cast<double>(cells) / 1e9) << "G"; + } else if(cells >= 1000000ull){ + oss << (static_cast<double>(cells) / 1e6) << "M"; + } else if(cells >= 1000ull){ + oss << (static_cast<double>(cells) / 1e3) << "k"; + } else { + oss << cells; + } + + return oss.str(); +} + +} // namespace impl + +/// Renders a snapshot as a fixed width table. +inline std::string render_table(const std::vector<worker_record> &records, + uint64_t now_ms){ + std::ostringstream oss; + + oss << " " << std::left + << std::setw(6) << "sess" + << std::setw(18) << "worker" + << std::setw(9) << "kind" + << std::setw(12) << "state" + << std::setw(9) << "rtt" + << std::setw(8) << "pongs" + << std::setw(8) << "seen" + << std::setw(8) << "cells" + << "detail" << "\n"; + + for(const auto &rec : records){ + std::ostringstream rtt; + if(rec.rtt_ewma_ms < 0.0){ + rtt << "-"; + } else { + rtt << std::fixed << std::setprecision(1) << rec.rtt_ewma_ms << "ms"; + } + + std::ostringstream seen; + const uint64_t age = (now_ms > rec.last_seen_ms) + ? (now_ms - rec.last_seen_ms) : 0u; + seen << (age / 1000u) << "s"; + + oss << " " << std::left + << std::setw(6) << rec.session_id + << std::setw(18) << (rec.worker_id.empty() ? "-" : rec.worker_id) + << std::setw(9) << (rec.kind.empty() ? "-" : rec.kind) + << std::setw(12) << to_string(rec.state) + << std::setw(9) << rtt.str() + << std::setw(8) << rec.pongs + << std::setw(8) << seen.str() + << std::setw(8) << impl::human_cells(rec.capacity_cells) + << rec.summary << "\n"; + } + + if(records.empty()){ + oss << " (no workers)\n"; + } + + return oss.str(); +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/run.hpp b/modules/swarm/c++/run.hpp new file mode 100644 index 0000000..b11bdf5 --- /dev/null +++ b/modules/swarm/c++/run.hpp @@ -0,0 +1,193 @@ +#pragma once + +#include "message.hpp" + +#include <kel/lbm/abstract/data.hpp> + +#include <cstdint> +#include <string> +#include <string_view> +#include <string_view> + +/** + * Run identity. + * + * There is none in the library today: outputs land in a directory named after + * the example, with `<field>_<step>.vtk` inside, so a rerun silently + * overwrites the one before it and nothing records what parameters produced + * what. A swarm makes that worse rather than better -- several workers write + * concurrently, and a result arriving for an unknown run has to be refusable. + * + * A run id is therefore the first thing the orchestrator mints and the thing + * every transfer names. It is sortable, unique, and safe as a directory name: + * + * 20261112T094501Z-a3f19c + * + * UTC so that two machines agree, second resolution because runs are not + * started faster than that by a human, and six hex digits of entropy so that + * two started in the same second by a script still differ. + */ +namespace kel { +namespace lbm { +namespace sch { + +using RunDescriptor = Struct< + Member<String, "run_id">, + Member<String, "name">, + Member<UInt64, "created_at_ms">, + Member<String, "schema_version">, + /** + * Whether halo exchange was on. False until milestone 3, and recorded + * rather than assumed: slices run with clamped edges until then, so the + * seams are wrong, and a stored result should say so rather than leave + * someone to mistake a seam artefact for a solver bug years later. + */ + Member<Bool, "halo_exchange">, + /// Opaque to everything here: whatever the admin endpoint was handed. + Member<String, "params_json"> +>; + +} // namespace sch + +namespace swarm { + +namespace impl { + +/** + * Civil date from a day count since 1970-01-01, by Howard Hinnant's algorithm. + * + * Deliberately not gmtime_r: this is pure arithmetic, so it needs no locale, + * no timezone database and no thread safety caveat, and it can be exercised + * directly in a test rather than through whatever the host libc happens to + * believe. It is also the same on a browser as on the orchestrator, which + * matters when both ends format or parse an id. + */ +struct civil_date { + int64_t year; + uint32_t month; ///< 1-12 + uint32_t day; ///< 1-31 +}; + +constexpr civil_date civil_from_days(int64_t z){ + z += 719468; + const int64_t era = (z >= 0 ? z : z - 146096) / 146097; + const uint64_t doe = static_cast<uint64_t>(z - era * 146097); + const uint64_t yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + const int64_t y = static_cast<int64_t>(yoe) + era * 400; + const uint64_t doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + const uint64_t mp = (5 * doy + 2) / 153; + const uint64_t d = doy - (153 * mp + 2) / 5 + 1; + const uint64_t m = mp + (mp < 10 ? 3 : -9); + return civil_date{y + (m <= 2 ? 1 : 0), static_cast<uint32_t>(m), + static_cast<uint32_t>(d)}; +} + +inline void append_padded(std::string &out, uint64_t value, uint32_t width){ + char buffer[20]; + uint32_t written = 0u; + do { + buffer[written++] = static_cast<char>('0' + (value % 10u)); + value /= 10u; + } while(value > 0u and written < sizeof(buffer)); + + for(uint32_t i = written; i < width; ++i){ + out.push_back('0'); + } + for(uint32_t i = written; i > 0u; --i){ + out.push_back(buffer[i - 1u]); + } +} + +inline void append_hex(std::string &out, uint64_t value, uint32_t digits){ + static constexpr char table[] = "0123456789abcdef"; + for(uint32_t i = digits; i > 0u; --i){ + out.push_back(table[(value >> ((i - 1u) * 4u)) & 0xfu]); + } +} + +} // namespace impl + +/** + * Formats a run id. + * + * Takes the clock and the entropy rather than reading them, for the same + * reason the registry takes `now_ms`: a test can then pin both and assert the + * exact string, instead of matching a pattern and hoping. + * + * Only the low 24 bits of `entropy` are used. + */ +inline std::string make_run_id(uint64_t now_ms, uint64_t entropy){ + const int64_t seconds = static_cast<int64_t>(now_ms / 1000u); + const int64_t days = (seconds >= 0) ? (seconds / 86400) + : ((seconds - 86399) / 86400); + const uint64_t secs_of_day = static_cast<uint64_t>(seconds - days * 86400); + + const auto date = impl::civil_from_days(days); + + std::string out; + out.reserve(24u); + + impl::append_padded(out, static_cast<uint64_t>(date.year), 4u); + impl::append_padded(out, date.month, 2u); + impl::append_padded(out, date.day, 2u); + out.push_back('T'); + impl::append_padded(out, secs_of_day / 3600u, 2u); + impl::append_padded(out, (secs_of_day / 60u) % 60u, 2u); + impl::append_padded(out, secs_of_day % 60u, 2u); + out.push_back('Z'); + out.push_back('-'); + impl::append_hex(out, entropy, 6u); + + return out; +} + +/** + * Whether a string could have come from make_run_id. + * + * The sink uses a run id as a directory name, so this is also the check that + * keeps a hostile id from escaping the storage root -- there is no '/', no + * '.', and nothing but a fixed shape of digits and hex. + */ +inline bool is_run_id(std::string_view id){ + if(id.size() != 23u){ + return false; + } + + auto digit = [&](std::size_t i){ return id[i] >= '0' and id[i] <= '9'; }; + auto hex = [&](std::size_t i){ + return (id[i] >= '0' and id[i] <= '9') or (id[i] >= 'a' and id[i] <= 'f'); + }; + + for(std::size_t i = 0u; i < 8u; ++i){ + if(not digit(i)){ return false; } + } + if(id[8] != 'T'){ return false; } + for(std::size_t i = 9u; i < 15u; ++i){ + if(not digit(i)){ return false; } + } + if(id[15] != 'Z'){ return false; } + if(id[16] != '-'){ return false; } + for(std::size_t i = 17u; i < 23u; ++i){ + if(not hex(i)){ return false; } + } + + return true; +} + +} // namespace swarm + +namespace sch { + +using MsgRunStart = Envelope<Member<RunDescriptor, "run">>; +using MsgRunEnd = Envelope<Member<String, "run_id">, Member<String, "outcome">>; + +} // namespace sch + +namespace run_outcome { +inline constexpr std::string_view finished = "finished"; +inline constexpr std::string_view cancelled = "cancelled"; +inline constexpr std::string_view failed = "failed"; +} // namespace run_outcome + +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/signal.hpp b/modules/swarm/c++/signal.hpp new file mode 100644 index 0000000..e8d76bd --- /dev/null +++ b/modules/swarm/c++/signal.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include <kel/lbm/abstract/schema.hpp> + +#include <string_view> + +/** + * WebRTC signalling, carried over the WebSocket that precedes the data channel. + * + * This is a different protocol from the control messages in message.hpp and + * deliberately so: signalling exists only to get a peer connection established, + * after which everything moves to the data channel. It lives here rather than + * in the orchestrator because both the browser worker and the native test + * worker have to speak it, and one definition is the only way they cannot + * disagree. + * + * The shapes mirror what a browser hands you, with one deliberate difference: + * an ICE candidate arrives from JS as {candidate, sdpMid, sdpMLineIndex} and is + * sent on as {type:"candidate", candidate, mid}. Normalising at the JS boundary + * keeps the wire uniform -- every message here is a struct with a "type" -- at + * the cost of three lines in the worker's transport glue. + */ +namespace kel { +namespace lbm { +namespace sch { + +/// Stage one of the same two stage decode the control protocol uses. +using SignalHeader = Struct<Member<String, "type">>; + +/// "offer" or "answer". +using SignalDescription = Struct< + Member<String, "type">, + Member<String, "sdp"> +>; + +/// "candidate". mid may be empty; libdatachannel treats that as unhinted. +using SignalCandidate = Struct< + Member<String, "type">, + Member<String, "candidate">, + Member<String, "mid"> +>; + +} // namespace sch + +namespace sig { +inline constexpr std::string_view offer = "offer"; +inline constexpr std::string_view answer = "answer"; +inline constexpr std::string_view candidate = "candidate"; +} // namespace sig + +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/c++/swarm.hpp b/modules/swarm/c++/swarm.hpp new file mode 100644 index 0000000..a7c066f --- /dev/null +++ b/modules/swarm/c++/swarm.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include "capability.hpp" +#include "capacity.hpp" +#include "interpret.hpp" +#include "message.hpp" +#include "run.hpp" +#include "probe_search.hpp" +#include "registry.hpp" +#include "signal.hpp" +#include "wire.hpp" diff --git a/modules/swarm/c++/wire.hpp b/modules/swarm/c++/wire.hpp new file mode 100644 index 0000000..7c7c7d0 --- /dev/null +++ b/modules/swarm/c++/wire.hpp @@ -0,0 +1,119 @@ +#pragma once + +#include "message.hpp" + +#include <kel/lbm/abstract/data.hpp> +#include <kel/lbm/abstract/json.hpp> + +#include <string> +#include <string_view> + +/** + * Encoding and decoding control messages. + * + * Everything here runs the JSON codec in strict mode. That is not the codec's + * default, and the default is wrong for a wire: codec_config<enc::Json>::strict + * is false so that config files can state only their deviations, which means a + * worker sending "{}" would decode cleanly into a default constructed struct. + * A registration that silently becomes all zeroes is precisely the failure this + * protocol must not have, so strict is set explicitly on every wire codec. + * + * Strict only governs *missing* members. Unknown members are skipped by the + * Struct decoder, which is what makes the two stage decode below work and what + * lets a newer worker talk to an older orchestrator. + */ +namespace kel { +namespace lbm { +namespace swarm { + +namespace impl { +template <typename Schema> +codec<Schema, enc::Json> make_wire_codec(){ + codec_config<enc::Json> cfg; + cfg.strict = true; + // Control messages nest at most capability -> webgl2 -> feasible -> plan, + // so the default guard against pathological input is already generous. + return codec<Schema, enc::Json>{cfg}; +} +} // namespace impl + +/** + * Serialise a message to the text that goes on the data channel. + */ +template <typename Schema> +error_or<std::string> encode_message(const data<Schema> &msg){ + auto wire_codec = impl::make_wire_codec<Schema>(); + + data<Schema, enc::Json> out; + auto eov = wire_codec.encode(msg, out); + if(eov.is_error()){ + return std::move(eov.get_error()); + } + + return out.stl_string(); +} + +/** + * Decode text into a known message type. Fails if any member is absent. + */ +template <typename Schema> +error_or<data<Schema>> decode_message(std::string_view text){ + auto wire_codec = impl::make_wire_codec<Schema>(); + + data<Schema, enc::Json> in{text}; + data<Schema> out; + auto eov = wire_codec.decode(in, out); + if(eov.is_error()){ + return std::move(eov.get_error()); + } + + return out; +} + +/** + * Stage one: read just the common members so the caller learns which type to + * decode. The payload is skipped rather than parsed into anything, so this + * succeeds for every message type and fails only on genuinely malformed input + * or a message missing the common members. + */ +inline error_or<data<sch::MsgHeader>> peek_header(std::string_view text){ + return decode_message<sch::MsgHeader>(text); +} + +/** + * Fills in the members every message shares. Kept in one place so a message + * built by the orchestrator and one built by the worker cannot disagree about + * the shape of the envelope. + */ +template <typename Schema> +void set_envelope(data<Schema> &msg, std::string_view type, + std::string_view worker_id, uint64_t seq){ + msg.template get<"type">().set(std::string{type}); + msg.template get<"worker_id">().set(std::string{worker_id}); + msg.template get<"seq">().set(seq); +} + +/** + * Convenience for the common shape: build a message, stamp the envelope. + */ +template <typename Schema> +data<Schema> make_message(std::string_view type, std::string_view worker_id, + uint64_t seq){ + data<Schema> msg; + set_envelope(msg, type, worker_id, seq); + return msg; +} + +/// Reads the "type" of an already decoded header. +inline std::string_view header_type(const data<sch::MsgHeader> &hdr){ + return hdr.template get<"type">().stl_view(); +} + +/// Reads the "worker_id" of an already decoded header. +inline std::string_view header_worker_id(const data<sch::MsgHeader> &hdr){ + return hdr.template get<"worker_id">().stl_view(); +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/modules/swarm/tests/SConscript b/modules/swarm/tests/SConscript new file mode 100644 index 0000000..d1b381e --- /dev/null +++ b/modules/swarm/tests/SConscript @@ -0,0 +1,32 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +test_cases_env = env.Clone(); + +test_cases_env.Append(LIBS=['forstio-test']); + +test_cases_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +test_cases_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += test_cases_env.sources; +env.headers += test_cases_env.headers; + +objects_static = [] +test_cases_env.add_source_files(objects_static, test_cases_env.sources, shared=False); +test_cases_env.program = test_cases_env.Program('#bin/tests', [objects_static]); +# , env.library_static]); + +# Set Alias +env.Alias('test', test_cases_env.program); +env.Alias('check', test_cases_env.program); + +env.targets += ['test','check']; diff --git a/modules/swarm/tests/capacity.cpp b/modules/swarm/tests/capacity.cpp new file mode 100644 index 0000000..c269aa1 --- /dev/null +++ b/modules/swarm/tests/capacity.cpp @@ -0,0 +1,234 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/capacity.hpp" + +namespace { +using namespace kel::lbm::swarm; + +/** + * The packing arithmetic is small enough that the interesting thing to test is + * not that it computes, but that the ceilings it implies are the ones we think + * they are. These are compile time assertions on purpose: they are statements + * about which descriptors are usable, and the answer should not depend on + * running the test on a particular device. + */ + +SAW_TEST("Packing / planes and idle slots per descriptor"){ + // D1Q3, D2Q5, D2Q9, D3Q19, D3Q27. + static_assert(packing_math::planes(3u) == 1u, "D1Q3 needs one texture"); + static_assert(packing_math::planes(5u) == 2u, "D2Q5 needs two textures"); + static_assert(packing_math::planes(9u) == 3u, "D2Q9 needs three textures"); + static_assert(packing_math::planes(19u) == 5u, "D3Q19 needs five textures"); + static_assert(packing_math::planes(27u) == 7u, "D3Q27 needs seven textures"); + + static_assert(packing_math::idle_slots(3u) == 1u, "D1Q3 wastes one slot"); + static_assert(packing_math::idle_slots(5u) == 3u, "D2Q5 wastes three slots"); + static_assert(packing_math::idle_slots(9u) == 3u, "D2Q9 wastes three slots"); + static_assert(packing_math::idle_slots(19u) == 1u, "D3Q19 wastes one slot"); + static_assert(packing_math::idle_slots(27u) == 1u, "D3Q27 wastes one slot"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +SAW_TEST("Packing / slots are a partition of the texels"){ + // planes*4 is exactly Q plus the waste, for every Q. Cheap, but it is the + // invariant every slot table depends on. + for(uint64_t q = 1u; q <= 64u; ++q){ + const uint64_t total = packing_math::planes(q) * packing_math::slots_per_texel; + SAW_EXPECT(total == q + packing_math::idle_slots(q), + "planes*4 must be Q plus the idle slots"); + SAW_EXPECT(packing_math::idle_slots(q) < packing_math::slots_per_texel, + "Waste below a full texel, or a plane was over allocated"); + } +} + +/** + * The limit that decides whether a descriptor is usable at all. WebGL2 + * guarantees only four colour attachments, so the 3D descriptors cannot write + * their whole distribution in one pass on a conformant minimum spec device. + * Asserting it here means we learn it from a test rather than from a browser. + */ +SAW_TEST("Packing / single pass only reaches Q <= 16 at guaranteed limits"){ + constexpr uint64_t mrt = webgl2_guaranteed::max_draw_buffers; + + static_assert(packing_math::fits_single_pass(9u, mrt), + "D2Q9 must fit one pass on any conformant WebGL2"); + static_assert(packing_math::fits_single_pass(5u, mrt), "D2Q5 must fit"); + static_assert(packing_math::fits_single_pass(3u, mrt), "D1Q3 must fit"); + + static_assert(not packing_math::fits_single_pass(19u, mrt), + "D3Q19 must not claim a single pass at guaranteed limits"); + static_assert(not packing_math::fits_single_pass(27u, mrt), + "D3Q27 must not claim a single pass at guaranteed limits"); + + static_assert(packing_math::passes(19u, mrt) == 2u, "D3Q19 needs two passes"); + static_assert(packing_math::passes(27u, mrt) == 2u, "D3Q27 needs two passes"); + + // Sixteen is the boundary: four planes, four attachments. + static_assert(packing_math::fits_single_pass(16u, mrt), "Q=16 is the last that fits"); + static_assert(not packing_math::fits_single_pass(17u, mrt), "Q=17 must not fit"); + + SAW_EXPECT(packing_math::passes(9u, 0u) == 0u, + "A device reporting no draw buffers must not divide by zero"); +} + +SAW_TEST("Packing / stuffing aux fields into idle slots"){ + // D2Q9 has three idle slots, so a lone info flag rides along free... + static_assert(packing_math::aux_planes(9u, 1u, true) == 0u, + "One aux scalar must fit D2Q9's idle slots"); + // ...but the four fields a PSM run wants do not. + static_assert(packing_math::aux_planes(9u, 4u, true) == 1u, + "Four aux scalars must not fit three idle slots"); + // Unstuffed always pays for its own plane. + static_assert(packing_math::aux_planes(9u, 1u, false) == 1u, + "Unstuffed aux must take a plane of its own"); + + // D3Q19 and D3Q27 have exactly one idle slot, so info is free there too. + static_assert(packing_math::aux_planes(19u, 1u, true) == 0u, "D3Q19 stuffs info"); + static_assert(packing_math::aux_planes(27u, 1u, true) == 0u, "D3Q27 stuffs info"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +SAW_TEST("Packing / bytes per cell"){ + // D2Q9, double buffered, info stuffed: 2*3 textures * 16 bytes. + static_assert(packing_math::bytes_per_cell(9u, 2u, 1u, true) == 96u, + "D2Q9 stuffed must be 96 bytes per cell"); + // Unstuffed adds a plane: 2*3 + 1 textures. + static_assert(packing_math::bytes_per_cell(9u, 2u, 1u, false) == 112u, + "D2Q9 unstuffed must be 112 bytes per cell"); + static_assert(packing_math::bytes_per_cell(19u, 2u, 1u, true) == 160u, + "D3Q19 stuffed must be 160 bytes per cell"); + static_assert(packing_math::bytes_per_cell(27u, 2u, 1u, true) == 224u, + "D3Q27 stuffed must be 224 bytes per cell"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +SAW_TEST("Capacity / integer square root is exact"){ + static_assert(isqrt(0u) == 0u, "isqrt(0)"); + static_assert(isqrt(1u) == 1u, "isqrt(1)"); + static_assert(isqrt(2u) == 1u, "isqrt(2)"); + static_assert(isqrt(3u) == 1u, "isqrt(3)"); + static_assert(isqrt(4u) == 2u, "isqrt(4)"); + + // The property, not just the samples: isqrt(n)^2 <= n < (isqrt(n)+1)^2. + for(uint64_t n = 0u; n < 4096u; ++n){ + const uint64_t r = isqrt(n); + SAW_EXPECT(r * r <= n, "isqrt must not overshoot"); + SAW_EXPECT((r + 1u) * (r + 1u) > n, "isqrt must not undershoot"); + } + + // Exact squares near the top of the range, where a float sqrt would drift. + const uint64_t big = 3037000499u; + SAW_EXPECT(isqrt(big * big) == big, "isqrt must stay exact for large squares"); + SAW_EXPECT(isqrt(big * big - 1u) == big - 1u, "isqrt must round down"); +} + +SAW_TEST("Capacity / the texture bound is the squared usable edge"){ + // Ghost eats a ring off each side, so the edge is S - 2G. + static_assert(capacity_math::texture_bound_cells_2d(2048u, 0u) == 2048u * 2048u, + "No ghost means the full edge"); + static_assert(capacity_math::texture_bound_cells_2d(2048u, 1u) == 2046u * 2046u, + "A one cell ring costs two off the edge"); + + // A device whose texture dimension cannot even hold the ring has no room. + static_assert(capacity_math::texture_bound_cells_2d(2u, 1u) == 0u, + "An edge consumed entirely by ghost leaves nothing"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +/** + * The headline number: how much lattice a worker can actually hold. The point + * of the test is the relationship between the two bounds, since on real + * hardware the texture bound is roughly a hundred times looser than memory and + * a capacity model that used it would be badly wrong. + */ +SAW_TEST("Capacity / memory binds before texture size on real devices"){ + constexpr uint64_t bpc = packing_math::bytes_per_cell(9u, 2u, 1u, true); + static_assert(bpc == 96u, "D2Q9 stuffed"); + + // A typical desktop: 16384 max texture, 512MB of budget. + constexpr uint64_t desktop_tex = 16384u; + constexpr uint64_t budget = 512ull * 1024ull * 1024ull; + + constexpr uint64_t by_texture = capacity_math::texture_bound_cells_2d(desktop_tex, 0u); + constexpr uint64_t by_memory = capacity_math::cells_from_bytes(budget, bpc); + + static_assert(by_memory < by_texture, + "Memory has to be the binding constraint on a typical desktop"); + static_assert(by_texture / by_memory > 40u, + "The texture bound should be looser by a wide margin, not a hair"); + + constexpr uint64_t cells = capacity_math::max_cells_2d(budget, bpc, desktop_tex, 0u); + static_assert(cells == by_memory, "max_cells must take the tighter bound"); + + SAW_EXPECT(cells * bpc <= budget, "The answer has to fit the budget it came from"); +} + +SAW_TEST("Capacity / minimum spec can be texture bound instead"){ + constexpr uint64_t bpc = packing_math::bytes_per_cell(9u, 2u, 1u, true); + + // Guaranteed limits with a generous budget: now the texture dimension is + // what runs out first, which is a different thing to report upstream. + constexpr uint64_t tex = webgl2_guaranteed::max_texture_size; + constexpr uint64_t budget = 4ull * 1024ull * 1024ull * 1024ull; + + constexpr uint64_t by_texture = capacity_math::texture_bound_cells_2d(tex, 1u); + constexpr uint64_t by_memory = capacity_math::cells_from_bytes(budget, bpc); + + static_assert(by_texture < by_memory, "Texture size must bind at minimum spec"); + static_assert(capacity_math::max_cells_2d(budget, bpc, tex, 1u) == by_texture, + "max_cells must take the tighter bound here too"); + + SAW_EXPECT(true, "static asserts carry this one"); +} + +SAW_TEST("Capacity / a fitted domain always fits"){ + constexpr uint64_t bpc = packing_math::bytes_per_cell(9u, 2u, 1u, true); + + // Sweep budgets and aspect ratios; the invariant is that the answer never + // exceeds the cell count it was given, and never exceeds the texture edge. + const uint64_t aspects[][2] = {{1u,1u}, {4u,1u}, {1u,4u}, {16u,9u}, {2u,1u}}; + const uint64_t tex = 16384u; + const uint64_t ghost = 1u; + const uint64_t edge = tex - 2u * ghost; + + for(uint64_t mb = 1u; mb <= 2048u; mb *= 2u){ + const uint64_t budget = mb * 1024ull * 1024ull; + const uint64_t cells = capacity_math::max_cells_2d(budget, bpc, tex, ghost); + + for(const auto& a : aspects){ + const auto dom = capacity_math::fit_domain_2d(cells, a[0], a[1], tex, ghost); + + SAW_EXPECT(dom.cells() <= cells, + "A fitted domain must not exceed the measured capacity"); + SAW_EXPECT(dom.nx <= edge and dom.ny <= edge, + "A fitted domain must not exceed the texture edge"); + SAW_EXPECT(dom.nx > 0u and dom.ny > 0u, + "A non zero budget must yield a non empty domain"); + SAW_EXPECT(dom.cells() * bpc <= budget, + "The domain must fit the byte budget it was derived from"); + } + } +} + +SAW_TEST("Capacity / fitting respects the requested aspect"){ + // 4:1 on a square-ish budget should come out roughly four times as wide as + // it is tall. Exactness is not the claim -- flooring twice costs a little -- + // but the shape has to be recognisably what was asked for. + const auto dom = capacity_math::fit_domain_2d(1024u * 256u, 4u, 1u, 16384u, 0u); + + SAW_EXPECT(dom.nx > dom.ny, "A 4:1 request must be wider than tall"); + const uint64_t ratio_x100 = (dom.nx * 100u) / dom.ny; + SAW_EXPECT(ratio_x100 >= 380u and ratio_x100 <= 420u, + "A 4:1 request must land near 4:1"); + + // Degenerate inputs are answered, not asserted on. + const auto zero = capacity_math::fit_domain_2d(0u, 4u, 1u, 16384u, 0u); + SAW_EXPECT(zero.cells() == 0u, "No capacity means no domain"); + const auto bad = capacity_math::fit_domain_2d(1024u, 0u, 1u, 16384u, 0u); + SAW_EXPECT(bad.cells() == 0u, "A degenerate aspect must not divide by zero"); +} +} diff --git a/modules/swarm/tests/interpret.cpp b/modules/swarm/tests/interpret.cpp new file mode 100644 index 0000000..53374f1 --- /dev/null +++ b/modules/swarm/tests/interpret.cpp @@ -0,0 +1,133 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/interpret.hpp" + +#include <string> + +namespace { +namespace sch { +using namespace kel::lbm::sch; +} +using namespace kel::lbm; +using namespace kel::lbm::swarm; + +/** + * The orchestrator's webgl2 branch cannot be reached by the native worker, and + * reaching it with a real browser means a human in the loop. These cover the + * part of it that is pure: turning a reported capability into a capacity and a + * summary line. + */ + +data<sch::WebGl2Capability> make_gl(uint64_t safe_bytes, uint64_t max_tex, + bool float_render = true){ + data<sch::WebGl2Capability> gl; + gl.get<"limits">().get<"max_texture_size">().set( + static_cast<uint32_t>(max_tex)); + gl.get<"limits">().get<"max_draw_buffers">().set(8u); + gl.get<"budget">().get<"probed_bytes_safe">().set(safe_bytes); + gl.get<"features">().get<"color_buffer_float">().set(float_render); + return gl; +} + +data<sch::NativeCapability> make_native(uint64_t memory, uint32_t threads, + bool sycl = false){ + data<sch::NativeCapability> nat; + nat.get<"threads">().set(threads); + nat.get<"memory_bytes">().set(memory); + nat.get<"cpu">().set(std::string{"x86_64"}); + nat.get<"has_sycl">().set(sycl); + return nat; +} + +SAW_TEST("Interpret / the reference packing is D2Q9 double buffered"){ + static_assert(reference_bytes_per_cell() == 96u, + "Capacity figures are quoted in 96 byte D2Q9 cells"); + SAW_EXPECT(true, "static assert carries this one"); +} + +SAW_TEST("Interpret / a browser's capacity comes from its measured budget"){ + // 512MiB safe budget on a roomy device: memory binds, not texture size. + const auto gl = make_gl(512ull * 1024ull * 1024ull, 16384u); + const uint64_t cells = webgl2_capacity_cells(gl); + + SAW_EXPECT(cells == (512ull * 1024ull * 1024ull) / 96u, + "Capacity is the budget divided by the reference cell size"); + SAW_EXPECT(cells * 96u <= 512ull * 1024ull * 1024ull, + "and must fit the budget it came from"); +} + +SAW_TEST("Interpret / a small texture dimension binds before memory"){ + // Generous budget, minimum spec texture size: the texture bound wins. + const auto gl = make_gl(4ull * 1024ull * 1024ull * 1024ull, 2048u); + const uint64_t cells = webgl2_capacity_cells(gl); + + SAW_EXPECT(cells == 2048ull * 2048ull, + "The texture dimension is the binding constraint here"); +} + +SAW_TEST("Interpret / a device that measured nothing has no capacity"){ + // What a worker reports when the probe failed outright -- the case the + // node self test produces, and the one a browser produces on a device + // with no float render target. + const auto gl = make_gl(0u, 16384u, false); + SAW_EXPECT(webgl2_capacity_cells(gl) == 0u, + "No measured budget means no capacity, not a guess from the limits"); +} + +SAW_TEST("Interpret / the summary says when a device cannot render float"){ + const auto usable = make_gl(256ull * 1024ull * 1024ull, 16384u, true); + const std::string ok_line = webgl2_summary(usable); + SAW_EXPECT(ok_line.find("tex 16384") != std::string::npos, + "The summary names the texture limit"); + SAW_EXPECT(ok_line.find("256.0 MiB") != std::string::npos, + "and the budget, in units a human reads"); + SAW_EXPECT(ok_line.find("NO FLOAT RENDER") == std::string::npos, + "A usable device is not flagged"); + + const auto broken = make_gl(0u, 16384u, false); + SAW_EXPECT(webgl2_summary(broken).find("NO FLOAT RENDER") != std::string::npos, + "A device with no float render target is flagged loudly, since that " + "is the difference between slow and impossible"); +} + +SAW_TEST("Interpret / a native worker's capacity leaves room for the machine"){ + const auto nat = make_native(32ull * 1024ull * 1024ull * 1024ull, 16u); + const uint64_t cells = native_capacity_cells(nat); + + // A quarter of memory, because the rest of the machine exists. + SAW_EXPECT(cells == (32ull * 1024ull * 1024ull * 1024ull / 4u) / 96u, + "Native capacity is a quarter of memory in reference cells"); + SAW_EXPECT(cells * 96u * 4u <= 32ull * 1024ull * 1024ull * 1024ull, + "and stays well inside what the machine has"); +} + +SAW_TEST("Interpret / native and browser capacities are comparable"){ + // The point of quoting both in the same unit: a registry holding a mix of + // kinds can rank them against each other. + const auto gl = make_gl(2ull * 1024ull * 1024ull * 1024ull, 16384u); + const auto nat = make_native(8ull * 1024ull * 1024ull * 1024ull, 8u); + + const uint64_t browser_cells = webgl2_capacity_cells(gl); + const uint64_t native_cells = native_capacity_cells(nat); + + // 2GiB measured vs 8GiB/4 = 2GiB planned: the same, which is the check + // that one unit really is being used for both. + SAW_EXPECT(browser_cells == native_cells, + "Equivalent budgets must produce equal capacities across kinds"); +} + +SAW_TEST("Interpret / the native summary reports what a process can know"){ + const std::string plain = native_summary( + make_native(8ull * 1024ull * 1024ull * 1024ull, 4u)); + SAW_EXPECT(plain.find("4 threads") != std::string::npos, "thread count"); + SAW_EXPECT(plain.find("8.0 GiB") != std::string::npos, "memory"); + SAW_EXPECT(plain.find("x86_64") != std::string::npos, "architecture"); + SAW_EXPECT(plain.find("sycl") == std::string::npos, + "and does not claim sycl when there is none"); + + const std::string with_sycl = native_summary( + make_native(8ull * 1024ull * 1024ull * 1024ull, 4u, true)); + SAW_EXPECT(with_sycl.find("sycl") != std::string::npos, + "but says so when there is"); +} +} diff --git a/modules/swarm/tests/probe_search.cpp b/modules/swarm/tests/probe_search.cpp new file mode 100644 index 0000000..42f852b --- /dev/null +++ b/modules/swarm/tests/probe_search.cpp @@ -0,0 +1,202 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/capacity.hpp" +#include "../c++/probe_search.hpp" + +#include <string> + +namespace { +using namespace kel::lbm::swarm; + +/** + * Runs a search against a simulated device whose real capacity is `truth`, + * returning the outcome. This is the whole point of keeping the strategy apart + * from the GL calls: a device that would take a browser and several seconds to + * interrogate is a predicate here. + */ +struct probe_outcome { + uint64_t best; + uint64_t trials; + bool hit_ceiling; +}; + +probe_outcome run_against(uint64_t truth, allocation_search::config cfg = {}){ + allocation_search search{cfg}; + + while(not search.done()){ + search.report(search.candidate() <= truth); + } + + return probe_outcome{search.best_cells(), search.trials(), + search.hit_ceiling()}; +} + +SAW_TEST("Probe / never reports more than the device can do"){ + // The property that matters most. Over-reporting means the orchestrator + // hands out a domain that does not fit, and the worker fails at the point + // where it is least convenient. + const uint64_t truths[] = { + 65536u, 100000u, 1000000u, 2097152u, 5600000u, 12345678u, + 99999999u, 1u << 28, + }; + + for(uint64_t truth : truths){ + const auto out = run_against(truth); + SAW_EXPECT(out.best <= truth, + "A probe must never claim more than the device allows"); + } +} + +SAW_TEST("Probe / gets close to the real capacity"){ + // Within a quarter is plenty: the figure is a planning estimate against a + // budget that shifts as other tabs come and go. + const uint64_t truths[] = { + 200000u, 1000000u, 3000000u, 5600000u, 22000000u, + }; + + for(uint64_t truth : truths){ + const auto out = run_against(truth); + SAW_EXPECT(out.best * 4u >= truth * 3u, + std::string{"Probe landed too far below the truth for "} + + std::to_string(truth)); + } +} + +SAW_TEST("Probe / converges in a handful of trials"){ + // Doubling then bisecting should cost a dozen or so attempts even across + // three orders of magnitude of device. + allocation_search::config cfg; + + for(uint64_t truth = 65536u; truth < (1ull << 30); truth *= 3u){ + const auto out = run_against(truth, cfg); + SAW_EXPECT(out.trials <= cfg.max_trials, + "The backstop must hold"); + SAW_EXPECT(out.trials <= 20u, + std::string{"Too many trials for truth "} + std::to_string(truth)); + } +} + +SAW_TEST("Probe / a device that cannot do the opening attempt reports zero"){ + allocation_search::config cfg; + cfg.start_cells = 65536u; + + const auto out = run_against(1000u, cfg); + SAW_EXPECT(out.best == 0u, + "Failing the smallest attempt must report no capacity, not a guess"); + SAW_EXPECT(out.trials == 1u, "and must not keep trying"); +} + +SAW_TEST("Probe / the texture ceiling is reported, not confused with memory"){ + allocation_search::config cfg; + cfg.ceiling_cells = 1000000u; + + // Memory is effectively unlimited here, so the ceiling is what stops it. + const auto out = run_against(1ull << 40, cfg); + SAW_EXPECT(out.best == cfg.ceiling_cells, "It reaches the ceiling exactly"); + SAW_EXPECT(out.hit_ceiling, + "and says the texture dimension stopped it, not memory"); + + // Whereas a device that runs out of memory first must not claim the + // ceiling stopped it -- those are different things to report upstream. + const auto mem = run_against(300000u, cfg); + SAW_EXPECT(not mem.hit_ceiling, + "Running out of memory below the ceiling is not a ceiling hit"); + SAW_EXPECT(mem.best <= 300000u, "and still must not over-report"); +} + +SAW_TEST("Probe / a ceiling below the opening attempt is clamped"){ + allocation_search::config cfg; + cfg.start_cells = 65536u; + cfg.ceiling_cells = 1024u; + + allocation_search search{cfg}; + SAW_EXPECT(search.candidate() == 1024u, + "The first attempt must not exceed the ceiling"); + + const auto out = run_against(1ull << 30, cfg); + SAW_EXPECT(out.best == 1024u, "and the answer is the ceiling"); + SAW_EXPECT(out.hit_ceiling, "reported as such"); +} + +SAW_TEST("Probe / a zero ceiling means no capacity at all"){ + allocation_search::config cfg; + cfg.ceiling_cells = 0u; + cfg.start_cells = 0u; + + allocation_search search{cfg}; + SAW_EXPECT(search.done(), "Nothing to try means the search is already over"); + SAW_EXPECT(search.best_cells() == 0u, "and the answer is zero"); +} + +SAW_TEST("Probe / reporting after completion is ignored"){ + allocation_search search{allocation_search::config{}}; + while(not search.done()){ + search.report(false); + } + + const uint64_t trials = search.trials(); + search.report(true); + SAW_EXPECT(search.trials() == trials, + "A late answer must not restart a finished search"); + SAW_EXPECT(search.best_cells() == 0u, "nor change the result"); +} + +/** + * A device that answers inconsistently -- which a real one can, since memory + * pressure moves while you are measuring -- must still terminate. + */ +SAW_TEST("Probe / an unreliable device still terminates"){ + allocation_search::config cfg; + + // Alternating yes/no regardless of size. + { + allocation_search search{cfg}; + bool flip = true; + uint64_t guard = 0u; + while(not search.done()){ + search.report(flip); + flip = not flip; + SAW_EXPECT(++guard < 1000u, "The search must terminate"); + } + SAW_EXPECT(search.trials() <= cfg.max_trials, "within the backstop"); + } + + // Always yes, unbounded device: the doubling has to stop somewhere. + { + allocation_search search{cfg}; + uint64_t guard = 0u; + while(not search.done()){ + search.report(true); + SAW_EXPECT(++guard < 1000u, "Unbounded success must still terminate"); + } + SAW_EXPECT(search.trials() <= cfg.max_trials, "within the backstop"); + } +} + +/** + * The probe answers in cells; the orchestrator wants a domain. Checking the + * two compose means the number that comes off a device is usable as it stands. + */ +SAW_TEST("Probe / the measured capacity feeds the domain fitting"){ + constexpr uint64_t bpc = packing_math::bytes_per_cell(9u, 2u, 1u, true); + const uint64_t max_tex = 16384u; + + // A device good for about 512MB of lattice. + const uint64_t truth = capacity_math::cells_from_bytes( + 512ull * 1024ull * 1024ull, bpc); + + allocation_search::config cfg; + cfg.ceiling_cells = capacity_math::texture_bound_cells_2d(max_tex, 0u); + + const auto out = run_against(truth, cfg); + SAW_EXPECT(out.best <= truth, "Measured capacity must not exceed the truth"); + SAW_EXPECT(not out.hit_ceiling, + "Memory binds well before the texture dimension on a device like this"); + + const auto dom = capacity_math::fit_domain_2d(out.best, 4u, 1u, max_tex, 0u); + SAW_EXPECT(dom.cells() <= out.best, + "The fitted domain must fit what was actually measured"); + SAW_EXPECT(dom.cells() * bpc <= 512ull * 1024ull * 1024ull, + "and therefore fit in the memory the device really had"); +} +} diff --git a/modules/swarm/tests/registry.cpp b/modules/swarm/tests/registry.cpp new file mode 100644 index 0000000..45ebab3 --- /dev/null +++ b/modules/swarm/tests/registry.cpp @@ -0,0 +1,298 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/message.hpp" +#include "../c++/registry.hpp" + +#include <string> + +namespace { +namespace sch { +using namespace kel::lbm::sch; +} +using namespace kel::lbm; +using namespace kel::lbm::swarm; + +/** + * The registry takes `now_ms` at every entry point rather than reading a clock, + * so these tests drive a worker from healthy to dropped in microseconds. Doing + * this against a real connection would mean sleeping through several heartbeat + * intervals and hoping the timing held. + */ + +data<sch::WorkerLive> some_live(){ + data<sch::WorkerLive> live; + live.get<"state">().set(std::string{wstate::ready}); + live.get<"uptime_ms">().set(0u); + live.get<"steps_done">().set(0u); + live.get<"last_error">().set(std::string{""}); + return live; +} + +/** + * Registers one worker into an existing registry. Fills in place rather than + * returning one: registry holds a mutex, so it is neither copyable nor movable. + */ +void register_one(registry ®){ + reg.add_session("w1", 1000u); + reg.mark_registered("w1", "alpha", std::string{wkind::native}, 1000u, + "test worker", 1000u); +} + +worker_record only(const registry ®){ + auto snap = reg.snapshot(); + return snap.empty() ? worker_record{} : snap.front(); +} + +SAW_TEST("Registry / a session starts in signalling and is not pinged"){ + registry reg; + reg.add_session("w1", 100u); + + SAW_EXPECT(reg.size() == 1u, "The session is tracked"); + SAW_EXPECT(only(reg).state == worker_state::signalling, + "A fresh session has not said hello yet"); + + // Crucially the thresholds must not apply before a hello. A worker still + // negotiating is not unhealthy, and counting misses against it would drop + // it before it ever had a channel to answer on. + for(uint64_t i = 0u; i < 20u; ++i){ + auto result = reg.tick(200u + i); + SAW_EXPECT(result.to_ping.empty(), "A signalling session is not pinged"); + SAW_EXPECT(result.dropped.empty(), "A signalling session is not dropped"); + } + SAW_EXPECT(reg.size() == 1u, "It is still there after many ticks"); +} + +SAW_TEST("Registry / registration records the capability"){ + registry reg; + register_one(reg); + const auto rec = only(reg); + + SAW_EXPECT(rec.state == worker_state::registered, "hello moves it on"); + SAW_EXPECT(rec.worker_id == "alpha", "The self reported name is kept"); + SAW_EXPECT(rec.kind == wkind::native, "The kind is kept"); + SAW_EXPECT(rec.capacity_cells == 1000u, "The capacity is kept"); + SAW_EXPECT(rec.session_id == "w1", "Keyed on the assigned id"); +} + +SAW_TEST("Registry / registering an unknown session fails rather than creating one"){ + registry reg; + SAW_EXPECT(not reg.mark_registered("ghost", "a", "native", 1u, "", 1u), + "A hello from a session that has gone must not resurrect it"); + SAW_EXPECT(reg.size() == 0u, "and must not invent a record"); +} + +SAW_TEST("Registry / a ping is outstanding until its pong"){ + registry reg; + register_one(reg); + + auto t1 = reg.tick(2000u); + SAW_EXPECT(t1.to_ping.size() == 1u, "One worker, one ping"); + SAW_EXPECT(t1.to_ping.front().seq == 1u, "Sequences start at one"); + SAW_EXPECT(only(reg).outstanding_pings == 1u, "The ping is outstanding"); + + reg.record_pong("w1", 1u, 2000u, some_live(), 2020u); + SAW_EXPECT(only(reg).outstanding_pings == 0u, "The pong clears it"); + SAW_EXPECT(only(reg).pongs == 1u, "and is counted"); +} + +SAW_TEST("Registry / round trip is measured against our own clock"){ + registry reg; + register_one(reg); + + reg.tick(2000u); + // The worker echoes the timestamp we sent, so no clock agreement needed. + reg.record_pong("w1", 1u, 2000u, some_live(), 2040u); + SAW_EXPECT(only(reg).rtt_ewma_ms == 40.0, + "The first sample seeds the average exactly"); + + // Second sample at 80ms: 0.25*80 + 0.75*40 = 50. + reg.tick(3000u); + reg.record_pong("w1", 2u, 3000u, some_live(), 3080u); + const double rtt = only(reg).rtt_ewma_ms; + SAW_EXPECT(rtt > 49.9 and rtt < 50.1, "Smoothing weights the history"); +} + +SAW_TEST("Registry / a clock that goes backwards does not produce a negative rtt"){ + registry reg; + register_one(reg); + + reg.tick(2000u); + // Echoed timestamp later than now. Should clamp, not underflow into a + // huge positive via unsigned subtraction. + reg.record_pong("w1", 1u, 5000u, some_live(), 2000u); + SAW_EXPECT(only(reg).rtt_ewma_ms == 0.0, "A backwards clock reads as zero"); +} + +/** + * The transition that actually matters, and the one that is tedious to provoke + * against a live connection. + */ +SAW_TEST("Registry / stale then dropped, at the configured thresholds"){ + worker_thresholds limits; + limits.stale_after = 2u; + limits.drop_after = 4u; + + registry reg{limits}; + register_one(reg); + uint64_t t = 2000u; + + // Beat 1 and 2: outstanding climbs, still considered healthy. + auto b1 = reg.tick(t += 100u); + SAW_EXPECT(b1.newly_stale.empty(), "One miss is not yet doubtful"); + SAW_EXPECT(only(reg).state == worker_state::registered, "still registered"); + + auto b2 = reg.tick(t += 100u); + SAW_EXPECT(b2.newly_stale.empty(), "Two outstanding, threshold not crossed yet"); + + // Beat 3 sees outstanding == 2, which meets stale_after. + auto b3 = reg.tick(t += 100u); + SAW_EXPECT(b3.newly_stale.size() == 1u, "Crossing into stale is reported once"); + SAW_EXPECT(b3.newly_stale.front() == "w1", "and names the session"); + SAW_EXPECT(only(reg).state == worker_state::stale, "and is recorded"); + + // Reported once, not every beat afterwards -- otherwise the log fills up. + auto b4 = reg.tick(t += 100u); + SAW_EXPECT(b4.newly_stale.empty(), "Stale is announced once, not repeatedly"); + SAW_EXPECT(only(reg).state == worker_state::stale, "and it stays stale"); + + // Beat 5 sees outstanding == 4, which meets drop_after. + auto b5 = reg.tick(t += 100u); + SAW_EXPECT(b5.dropped.size() == 1u, "It is dropped at the threshold"); + SAW_EXPECT(b5.dropped.front().worker_id == "alpha", + "The dropped record is handed back so the caller can log it"); + SAW_EXPECT(reg.size() == 0u, "and removed from the registry"); + SAW_EXPECT(b5.to_ping.empty(), "A dropped session is not also pinged"); +} + +SAW_TEST("Registry / answering again clears stale"){ + worker_thresholds limits; + limits.stale_after = 2u; + limits.drop_after = 10u; + + registry reg{limits}; + register_one(reg); + uint64_t t = 2000u; + + reg.tick(t += 100u); + reg.tick(t += 100u); + auto b3 = reg.tick(t += 100u); + SAW_EXPECT(only(reg).state == worker_state::stale, "It went stale"); + + // The worker comes back, answering the ping that is actually outstanding. + reg.record_pong("w1", b3.to_ping.front().seq, t, some_live(), t + 5u); + + SAW_EXPECT(only(reg).state == worker_state::registered, + "Answering clears the doubt"); + SAW_EXPECT(only(reg).outstanding_pings == 0u, "and resets the miss count"); +} + +SAW_TEST("Registry / recovery does not silently restore readiness"){ + // Readiness is something only the worker can assert, so a recovered worker + // goes back to registered and has to say 'ready' again. + worker_thresholds limits; + limits.stale_after = 1u; + limits.drop_after = 10u; + + registry reg{limits}; + register_one(reg); + reg.mark_ready("w1", some_live(), 1500u); + SAW_EXPECT(only(reg).state == worker_state::ready, "It was ready"); + + reg.tick(2000u); + auto b2 = reg.tick(2100u); + SAW_EXPECT(only(reg).state == worker_state::stale, "Then it went quiet"); + + reg.record_pong("w1", b2.to_ping.front().seq, 2100u, some_live(), 2110u); + SAW_EXPECT(only(reg).state == worker_state::registered, + "Recovery must not reassert readiness on the worker's behalf"); +} + +/** + * A worker consistently one beat behind would otherwise look perfectly healthy: + * every tick adds a miss, every late pong clears it, and the miss count never + * reaches the threshold. + */ +SAW_TEST("Registry / a stale sequence pong does not clear the miss count"){ + registry reg; + register_one(reg); + + auto b1 = reg.tick(2000u); + const uint64_t old_seq = b1.to_ping.front().seq; + + reg.tick(2100u); + reg.tick(2200u); + SAW_EXPECT(only(reg).outstanding_pings == 3u, "Three pings outstanding"); + + // Answer the first one, long after it was superseded. + reg.record_pong("w1", old_seq, 2000u, some_live(), 2250u); + + SAW_EXPECT(only(reg).outstanding_pings == 3u, + "A pong for a superseded ping must not clear the misses"); + SAW_EXPECT(only(reg).pongs == 0u, "nor count as an answer"); + SAW_EXPECT(only(reg).last_seen_ms == 2250u, + "but it does prove the channel is alive"); +} + +SAW_TEST("Registry / removal returns the record once"){ + registry reg; + register_one(reg); + + auto gone = reg.remove("w1"); + SAW_EXPECT(gone.has_value(), "Removing returns what was removed"); + SAW_EXPECT(gone->worker_id == "alpha", "with its contents intact"); + SAW_EXPECT(reg.size() == 0u, "and empties the registry"); + + // Idempotent: several teardown paths converge on this and whichever + // arrives first should win without the others erroring. + SAW_EXPECT(not reg.remove("w1").has_value(), "A second removal is a no-op"); +} + +SAW_TEST("Registry / many workers are ticked independently"){ + worker_thresholds limits; + limits.stale_after = 2u; + limits.drop_after = 3u; + + registry reg{limits}; + for(uint64_t i = 0u; i < 5u; ++i){ + const std::string id = "w" + std::to_string(i); + reg.add_session(id, 1000u); + reg.mark_registered(id, "worker" + std::to_string(i), "native", + 1000u * i, "", 1000u); + } + + uint64_t t = 1000u; + for(int beat = 0; beat < 3; ++beat){ + auto result = reg.tick(t += 100u); + // w3 keeps answering; everyone else goes quiet. + for(const auto &target : result.to_ping){ + if(target.session_id == "w3"){ + reg.record_pong("w3", target.seq, t, some_live(), t + 1u); + } + } + } + + auto result = reg.tick(t += 100u); + SAW_EXPECT(result.dropped.size() == 4u, "The four silent workers are dropped"); + SAW_EXPECT(reg.size() == 1u, "The responsive one survives"); + SAW_EXPECT(only(reg).session_id == "w3", "and it is the right one"); +} + +SAW_TEST("Registry / the table renders without a worker and with one"){ + registry reg; + SAW_EXPECT(render_table(reg.snapshot(), 0u).find("(no workers)") + != std::string::npos, "An empty registry says so"); + + registry one; + register_one(one); + one.tick(2000u); + one.record_pong("w1", 1u, 2000u, some_live(), 2010u); + + const std::string table = render_table(one.snapshot(), 3000u); + SAW_EXPECT(table.find("alpha") != std::string::npos, "The worker is listed"); + SAW_EXPECT(table.find("native") != std::string::npos, "with its kind"); + SAW_EXPECT(table.find("10.0ms") != std::string::npos, "and its round trip"); + // 1000 cells renders as a bare count, not scaled. + SAW_EXPECT(table.find("1.0k") != std::string::npos, + "and a human readable capacity"); +} +} diff --git a/modules/swarm/tests/run.cpp b/modules/swarm/tests/run.cpp new file mode 100644 index 0000000..b7cd8f5 --- /dev/null +++ b/modules/swarm/tests/run.cpp @@ -0,0 +1,158 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/run.hpp" +#include "../c++/wire.hpp" + +#include <set> +#include <string> + +namespace { +namespace sch { +using namespace kel::lbm::sch; +} +using namespace kel::lbm; +using namespace kel::lbm::swarm; + +/// 2026-11-12T09:45:01Z, as milliseconds since the epoch. +constexpr uint64_t reference_ms = 1794476701000ull; + +SAW_TEST("Run / the id has the documented shape"){ + // Pinned exactly rather than pattern matched: make_run_id takes its clock + // and entropy, so the whole string is determined and a drift in either + // half should fail loudly. + const std::string id = make_run_id(reference_ms, 0xa3f19cull); + SAW_EXPECT(id == "20261112T094501Z-a3f19c", + std::string{"unexpected run id: "} + id); + SAW_EXPECT(id.size() == 23u, "and the fixed width the sink relies on"); +} + +SAW_TEST("Run / only the low 24 bits of entropy are used"){ + const std::string a = make_run_id(reference_ms, 0xffffffffffffffffull); + SAW_EXPECT(a == "20261112T094501Z-ffffff", "the top bits are discarded"); + + const std::string b = make_run_id(reference_ms, 0x0ull); + SAW_EXPECT(b == "20261112T094501Z-000000", "and zero pads rather than shortens"); +} + +SAW_TEST("Run / ids sort chronologically as plain strings"){ + // The point of the format: `ls` and a std::map give time order for free, + // with no parsing anywhere. + const std::string earlier = make_run_id(reference_ms, 0xffffffull); + const std::string later = make_run_id(reference_ms + 1000u, 0x000000ull); + + SAW_EXPECT(earlier < later, + "A later run must sort after an earlier one even with lower entropy"); + + // Across a month and a year boundary, where a naive format breaks. + SAW_EXPECT(make_run_id(reference_ms, 0u) + < make_run_id(reference_ms + 30ull * 86400000ull, 0u), + "and across a month"); +} + +SAW_TEST("Run / the civil date conversion handles the awkward cases"){ + using kel::lbm::swarm::impl::civil_from_days; + + // The epoch itself. + { + const auto d = civil_from_days(0); + SAW_EXPECT(d.year == 1970 and d.month == 1u and d.day == 1u, "epoch"); + } + // A leap day, which is where a hand-rolled conversion usually goes wrong. + { + const auto d = civil_from_days(19782); // 2024-02-29 + SAW_EXPECT(d.year == 2024 and d.month == 2u and d.day == 29u, + "2024 is a leap year and the 29th exists"); + } + // 2000 was a leap year despite being a century. + { + const auto d = civil_from_days(11016); // 2000-02-29 + SAW_EXPECT(d.year == 2000 and d.month == 2u and d.day == 29u, + "the 400 year rule"); + } + // The day after the 2024 leap day, so an off-by-one shows. + { + const auto d = civil_from_days(19783); + SAW_EXPECT(d.year == 2024 and d.month == 3u and d.day == 1u, "march 1st"); + } +} + +SAW_TEST("Run / midnight and the last second of a day format correctly"){ + // 2026-11-12T00:00:00Z + const uint64_t midnight = (reference_ms / 86400000ull) * 86400000ull; + SAW_EXPECT(make_run_id(midnight, 0u) == "20261112T000000Z-000000", + "midnight is all zeroes, not blank"); + + SAW_EXPECT(make_run_id(midnight + 86399000ull, 0u) + == "20261112T235959Z-000000", "and the last second is 235959"); + + // One more second rolls the date rather than producing 24:00:00. + SAW_EXPECT(make_run_id(midnight + 86400000ull, 0u) + == "20261113T000000Z-000000", "which rolls into the next day"); +} + +/** + * The sink uses a run id as a directory name, so this is the check that stops + * a hostile id escaping the storage root. + */ +SAW_TEST("Run / validation rejects anything that is not the exact shape"){ + SAW_EXPECT(is_run_id(make_run_id(reference_ms, 0xabcdefull)), + "what make_run_id produces must validate"); + + const char *bad[] = { + "", + "20261112T094501Z", // no entropy + "20261112T094501Z-a3f19", // one short + "20261112T094501Z-a3f19cd", // one long + "20261112t094501Z-a3f19c", // lowercase separator + "20261112T094501Z-A3F19C", // uppercase hex + "20261112T094501Z-a3f19g", // 'g' is not hex + "2026111 T094501Z-a3f19c", // space + "../../etc/passwd000000Z-a3f19c", // the reason this function exists + "20261112T094501Z/a3f19c", // a separator that is a path one + }; + + for(const char *id : bad){ + SAW_EXPECT(not is_run_id(id), + std::string{"must be rejected: "} + id); + } +} + +SAW_TEST("Run / distinct entropy gives distinct ids within one second"){ + std::set<std::string> seen; + for(uint64_t e = 0u; e < 512u; ++e){ + seen.insert(make_run_id(reference_ms, e)); + } + SAW_EXPECT(seen.size() == 512u, + "Two runs started in the same second must not collide"); +} + +SAW_TEST("Run / a run descriptor round trips on the wire"){ + auto start = make_message<sch::MsgRunStart>(msg::run_start, "orch", 0u); + + auto &run = start.get<"run">(); + run.get<"run_id">().set(make_run_id(reference_ms, 0xa3f19cull)); + run.get<"name">().set(std::string{"poiseuille 1024x256"}); + run.get<"created_at_ms">().set(reference_ms); + run.get<"schema_version">().set(std::string{schema_version}); + run.get<"halo_exchange">().set(false); + run.get<"params_json">().set(std::string{R"({"nx":1024,"ny":256})"}); + + auto eo_text = encode_message<sch::MsgRunStart>(start); + SAW_EXPECT(not eo_text.is_error(), "a run start encodes"); + + auto eo_hdr = peek_header(eo_text.get_value()); + SAW_EXPECT(not eo_hdr.is_error(), "and routes"); + SAW_EXPECT(header_type(eo_hdr.get_value()) == msg::run_start, "as a run start"); + + auto eo_back = decode_message<sch::MsgRunStart>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "and decodes"); + + const auto &back = eo_back.get_value().get<"run">(); + SAW_EXPECT(back.get<"run_id">().stl_view() == "20261112T094501Z-a3f19c", + "the id survives"); + SAW_EXPECT(not back.get<"halo_exchange">().get(), + "and the record that the seams were clamped survives with it"); + SAW_EXPECT(back.get<"params_json">().stl_view() == R"({"nx":1024,"ny":256})", + "the opaque params survive being nested inside JSON"); +} +} diff --git a/modules/swarm/tests/signal.cpp b/modules/swarm/tests/signal.cpp new file mode 100644 index 0000000..fc12193 --- /dev/null +++ b/modules/swarm/tests/signal.cpp @@ -0,0 +1,82 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/signal.hpp" +#include "../c++/wire.hpp" + +namespace { +namespace sch { +using namespace kel::lbm::sch; +} +using namespace kel::lbm; +using namespace kel::lbm::swarm; + +SAW_TEST("Signal / an offer round trips"){ + data<sch::SignalDescription> desc; + desc.get<"type">().set(std::string{sig::offer}); + desc.get<"sdp">().set(std::string{"v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\n"}); + + auto eo_text = encode_message<sch::SignalDescription>(desc); + SAW_EXPECT(not eo_text.is_error(), "An offer must encode"); + + auto eo_back = decode_message<sch::SignalDescription>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "An offer must decode"); + SAW_EXPECT(eo_back.get_value().get<"type">().stl_view() == sig::offer, + "type survives"); + // SDP is multi-line and full of characters JSON has to escape; that it + // comes back byte for byte is the only interesting property here. + SAW_EXPECT(eo_back.get_value().get<"sdp">().stl_view() + == "v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\n", + "SDP must survive escaping unchanged"); +} + +SAW_TEST("Signal / a candidate round trips"){ + data<sch::SignalCandidate> cand; + cand.get<"type">().set(std::string{sig::candidate}); + cand.get<"candidate">().set(std::string{"candidate:1 1 UDP 100 10.0.0.1 5000 typ host"}); + cand.get<"mid">().set(std::string{"0"}); + + auto eo_text = encode_message<sch::SignalCandidate>(cand); + SAW_EXPECT(not eo_text.is_error(), "A candidate must encode"); + + auto eo_back = decode_message<sch::SignalCandidate>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "A candidate must decode"); + SAW_EXPECT(eo_back.get_value().get<"mid">().stl_view() == "0", "mid survives"); +} + +SAW_TEST("Signal / the header routes without knowing the payload"){ + data<sch::SignalDescription> desc; + desc.get<"type">().set(std::string{sig::answer}); + desc.get<"sdp">().set(std::string{"v=0"}); + auto eo_text = encode_message<sch::SignalDescription>(desc); + SAW_EXPECT(not eo_text.is_error(), "encodes"); + + auto eo_hdr = decode_message<sch::SignalHeader>(eo_text.get_value()); + SAW_EXPECT(not eo_hdr.is_error(), "A header read must skip the sdp"); + SAW_EXPECT(eo_hdr.get_value().get<"type">().stl_view() == sig::answer, + "Header reports answer"); +} + +SAW_TEST("Signal / an empty mid is carried rather than dropped"){ + // libdatachannel accepts an unhinted candidate, so the empty string has to + // survive the round trip as an empty string and not become a decode error. + data<sch::SignalCandidate> cand; + cand.get<"type">().set(std::string{sig::candidate}); + cand.get<"candidate">().set(std::string{"candidate:1 1 UDP 100 10.0.0.1 5000 typ host"}); + cand.get<"mid">().set(std::string{""}); + + auto eo_text = encode_message<sch::SignalCandidate>(cand); + SAW_EXPECT(not eo_text.is_error(), "encodes"); + auto eo_back = decode_message<sch::SignalCandidate>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "An empty mid must still decode"); + SAW_EXPECT(eo_back.get_value().get<"mid">().stl_view().empty(), + "An empty mid stays empty"); +} + +SAW_TEST("Signal / a candidate missing its mid is rejected"){ + // Strictness applies here too: a signalling message that lost a member in + // transit should fail loudly rather than add a candidate with no mid. + auto eo = decode_message<sch::SignalCandidate>( + R"({"type":"candidate","candidate":"candidate:1 1 UDP 100 10.0.0.1 5000 typ host"})"); + SAW_EXPECT(eo.is_error(), "A missing mid must be an error"); +} +} diff --git a/modules/swarm/tests/wire.cpp b/modules/swarm/tests/wire.cpp new file mode 100644 index 0000000..5238619 --- /dev/null +++ b/modules/swarm/tests/wire.cpp @@ -0,0 +1,381 @@ +#include <forstio/test/suite.hpp> + +#include "../c++/wire.hpp" + +#include <string> + +namespace { +namespace sch { +using namespace kel::lbm::sch; +} +using namespace kel::lbm; +using namespace kel::lbm::swarm; + +/** + * Builds a filled-in live block. Every member is set, because the wire codecs + * run strict and a half built message is exactly what these tests are trying + * to catch elsewhere. + */ +data<sch::WorkerLive> make_live(){ + data<sch::WorkerLive> live; + live.get<"state">().set(std::string{wstate::ready}); + live.get<"uptime_ms">().set(1234u); + live.get<"steps_done">().set(42u); + live.get<"last_error">().set(std::string{""}); + return live; +} + +SAW_TEST("Wire / pong round trips unchanged"){ + auto msg = make_message<sch::MsgPong>(msg::pong, "worker-7", 19u); + msg.get<"sent_at_ms">().set(1000u); + msg.get<"worker_now_ms">().set(2000u); + msg.get<"live">() = make_live(); + + auto eo_text = encode_message<sch::MsgPong>(msg); + SAW_EXPECT(not eo_text.is_error(), "Encoding a complete message must succeed"); + const std::string text = eo_text.get_value(); + + auto eo_back = decode_message<sch::MsgPong>(text); + SAW_EXPECT(not eo_back.is_error(), "Decoding our own output must succeed"); + auto& back = eo_back.get_value(); + + SAW_EXPECT(back.get<"type">().stl_view() == msg::pong, "type survives"); + SAW_EXPECT(back.get<"worker_id">().stl_view() == "worker-7", "worker_id survives"); + SAW_EXPECT(back.get<"seq">().get() == 19u, "seq survives"); + SAW_EXPECT(back.get<"sent_at_ms">().get() == 1000u, "echoed timestamp survives"); + SAW_EXPECT(back.get<"worker_now_ms">().get() == 2000u, "worker clock survives"); + SAW_EXPECT(back.get<"live">().get<"steps_done">().get() == 42u, + "Nested struct members survive"); + SAW_EXPECT(back.get<"live">().get<"state">().stl_view() == wstate::ready, + "Nested strings survive"); +} + +SAW_TEST("Wire / welcome and ping round trip"){ + auto wel = make_message<sch::MsgWelcome>(msg::welcome, "worker-1", 0u); + wel.get<"heartbeat_ms">().set(2000u); + wel.get<"schema_version">().set(std::string{schema_version}); + + auto eo_text = encode_message<sch::MsgWelcome>(wel); + SAW_EXPECT(not eo_text.is_error(), "welcome encodes"); + auto eo_back = decode_message<sch::MsgWelcome>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "welcome decodes"); + SAW_EXPECT(eo_back.get_value().get<"heartbeat_ms">().get() == 2000u, + "heartbeat survives"); + + auto png = make_message<sch::MsgPing>(msg::ping, "worker-1", 5u); + png.get<"sent_at_ms">().set(999u); + auto eo_ptext = encode_message<sch::MsgPing>(png); + SAW_EXPECT(not eo_ptext.is_error(), "ping encodes"); + auto eo_pback = decode_message<sch::MsgPing>(eo_ptext.get_value()); + SAW_EXPECT(not eo_pback.is_error(), "ping decodes"); + SAW_EXPECT(eo_pback.get_value().get<"sent_at_ms">().get() == 999u, + "ping timestamp survives"); +} + +/** + * The two stage decode is the whole reason this protocol works without a + * tagged union: reading the header out of a full message has to succeed for + * every message type, skipping whatever payload it does not know about. + */ +SAW_TEST("Wire / the header can be read out of any message"){ + auto png = make_message<sch::MsgPing>(msg::ping, "w-a", 1u); + png.get<"sent_at_ms">().set(7u); + auto eo_png = encode_message<sch::MsgPing>(png); + SAW_EXPECT(not eo_png.is_error(), "ping encodes"); + + auto eo_hdr = peek_header(eo_png.get_value()); + SAW_EXPECT(not eo_hdr.is_error(), + "Reading a header out of a ping must skip the payload, not fail"); + SAW_EXPECT(header_type(eo_hdr.get_value()) == msg::ping, "Header reports ping"); + SAW_EXPECT(header_worker_id(eo_hdr.get_value()) == "w-a", "Header reports the id"); + + auto pong = make_message<sch::MsgPong>(msg::pong, "w-b", 2u); + pong.get<"sent_at_ms">().set(1u); + pong.get<"worker_now_ms">().set(2u); + pong.get<"live">() = make_live(); + auto eo_pong = encode_message<sch::MsgPong>(pong); + SAW_EXPECT(not eo_pong.is_error(), "pong encodes"); + + auto eo_hdr2 = peek_header(eo_pong.get_value()); + SAW_EXPECT(not eo_hdr2.is_error(), + "A nested struct payload must also be skippable"); + SAW_EXPECT(header_type(eo_hdr2.get_value()) == msg::pong, "Header reports pong"); + SAW_EXPECT(header_worker_id(eo_hdr2.get_value()) == "w-b", "Header reports the id"); +} + +/** + * The failure this protocol must not have. codec_config<enc::Json>::strict + * defaults to false so that config files can omit their defaults; on a wire + * that would let a worker send "{}" and have it decode into a silently zeroed + * registration. + */ +SAW_TEST("Wire / an empty object is rejected, not defaulted"){ + auto eo = decode_message<sch::MsgPing>("{}"); + SAW_EXPECT(eo.is_error(), + "An empty object must not decode into a default constructed message"); +} + +SAW_TEST("Wire / a missing member is rejected at every nesting depth"){ + // Top level: seq absent. + { + auto eo = decode_message<sch::MsgPing>( + R"({"type":"ping","worker_id":"w","sent_at_ms":1})"); + SAW_EXPECT(eo.is_error(), "A missing top level member must fail"); + } + // One level down: live is present but incomplete. This is the case that + // matters most, since strictness has to reach into nested structs or a + // half filled capability would register cleanly. + { + auto eo = decode_message<sch::MsgPong>( + R"({"type":"pong","worker_id":"w","seq":1,"sent_at_ms":1,)" + R"("worker_now_ms":2,"live":{"state":"ready"}})"); + SAW_EXPECT(eo.is_error(), "A missing nested member must fail"); + } + // And the complete form of the same message must still pass, so the test + // above is not just rejecting everything. + { + auto eo = decode_message<sch::MsgPong>( + R"({"type":"pong","worker_id":"w","seq":1,"sent_at_ms":1,)" + R"("worker_now_ms":2,"live":{"state":"ready","uptime_ms":0,)" + R"("steps_done":0,"last_error":""}})"); + SAW_EXPECT(not eo.is_error(), "The complete message must still decode"); + } +} + +/** + * Forward compatibility in the direction that works: a newer worker sending + * members an older orchestrator has never heard of still registers. + */ +SAW_TEST("Wire / unknown members are skipped"){ + auto eo = decode_message<sch::MsgPing>( + R"({"type":"ping","worker_id":"w","seq":3,"sent_at_ms":8,)" + R"("future_field":{"nested":[1,2,3]},"another":"x"})"); + SAW_EXPECT(not eo.is_error(), + "A newer sender's extra members must be skipped, not rejected"); + SAW_EXPECT(eo.get_value().get<"seq">().get() == 3u, "Known members still land"); + SAW_EXPECT(eo.get_value().get<"sent_at_ms">().get() == 8u, "and keep their values"); +} + +SAW_TEST("Wire / malformed input errors rather than defaulting"){ + const char* bad[] = { + "", + "{", + R"({"type":"ping","worker_id":"w","seq":1,"sent_at_ms":1)", // truncated + R"({"type":"ping","worker_id":"w","seq":1,"sent_at_ms":1}})", // trailing + R"({"type":"ping","worker_id":"w","seq":"not a number","sent_at_ms":1})", + "[1,2,3]", + "null", + }; + + for(const char* text : bad){ + auto eo = decode_message<sch::MsgPing>(text); + SAW_EXPECT(eo.is_error(), std::string{"Malformed input must error: "} + text); + } +} + +/** + * The capability envelope is the largest thing on the wire and the one whose + * shape has to survive being read by a process built from different source. + */ +SAW_TEST("Wire / hello carries a capability round trip"){ + auto hello = make_message<sch::MsgHelloWebGl2>(msg::hello, "worker-9", 0u); + + auto& cap = hello.get<"capability">(); + cap.get<"worker_id">().set(std::string{"worker-9"}); + cap.get<"kind">().set(std::string{wkind::webgl2}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(1700000000000u); + cap.get<"user_agent">().set(std::string{"test-agent"}); + + auto& gl = cap.get<"webgl2">(); + + auto& lim = gl.get<"limits">(); + lim.get<"max_texture_size">().set(16384u); + lim.get<"max_3d_texture_size">().set(2048u); + lim.get<"max_array_texture_layers">().set(2048u); + lim.get<"max_draw_buffers">().set(8u); + lim.get<"max_color_attachments">().set(8u); + lim.get<"max_texture_image_units">().set(16u); + lim.get<"max_combined_texture_image_units">().set(32u); + lim.get<"max_renderbuffer_size">().set(16384u); + lim.get<"max_fragment_uniform_vectors">().set(224u); + lim.get<"max_viewport_dims">().at({{0u}}).set(16384u); + lim.get<"max_viewport_dims">().at({{1u}}).set(16384u); + + auto& feat = gl.get<"features">(); + feat.get<"webgl2">().set(true); + feat.get<"offscreen_canvas">().set(true); + feat.get<"color_buffer_float">().set(true); + feat.get<"color_buffer_half_float">().set(true); + feat.get<"texture_float_linear">().set(false); + feat.get<"float_blend">().set(false); + feat.get<"parallel_shader_compile">().set(true); + feat.get<"disjoint_timer_query">().set(false); + + auto& bud = gl.get<"budget">(); + bud.get<"probed_bytes_max">().set(536870912u); + bud.get<"probed_bytes_safe">().set(322122547u); + bud.get<"probe_trials">().set(14u); + bud.get<"context_lost_during_probe">().set(false); + bud.get<"probe_hit_texture_size_bound">().set(false); + bud.get<"safety_factor">().set(0.6f); + + // One feasible plan, enough to prove the array of nested structs survives. + { + data<sch::PackingPlan> plan; + plan.get<"layout">().set(std::string{"plain2d"}); + plan.get<"aux_mode">().set(std::string{"stuffed"}); + plan.get<"d">().set(2u); + plan.get<"q">().set(9u); + plan.get<"ghost">().set(0u); + plan.get<"storage_bits">().set(32u); + plan.get<"slots_per_texel">().set(4u); + plan.get<"textures_df">().set(3u); + plan.get<"textures_aux">().set(0u); + plan.get<"states">().set(2u); + plan.get<"passes">().set(1u); + plan.get<"idle_slots">().set(3u); + plan.get<"domain">().at({{0u}}).set(1024u); + plan.get<"domain">().at({{1u}}).set(256u); + plan.get<"domain">().at({{2u}}).set(1u); + plan.get<"tiles">().at({{0u}}).set(1u); + plan.get<"tiles">().at({{1u}}).set(1u); + plan.get<"tiles_pow2">().set(true); + plan.get<"texture_size">().at({{0u}}).set(1024u); + plan.get<"texture_size">().at({{1u}}).set(256u); + plan.get<"bytes_total">().set(1024u * 256u * 96u); + plan.get<"bytes_useful">().set(1024u * 256u * 96u); + + for(uint64_t i = 0u; i < 9u; ++i){ + auto eo = plan.get<"slot_tex_of_q">().add(data<sch::UInt8>{static_cast<uint8_t>(i / 4u)}); + SAW_EXPECT(not eo.is_error(), "slot_tex_of_q grows"); + auto eo2 = plan.get<"slot_cmp_of_q">().add(data<sch::UInt8>{static_cast<uint8_t>(i % 4u)}); + SAW_EXPECT(not eo2.is_error(), "slot_cmp_of_q grows"); + } + auto eo3 = plan.get<"slot_tex_of_aux">().add(data<sch::UInt8>{2u}); + SAW_EXPECT(not eo3.is_error(), "slot_tex_of_aux grows"); + auto eo4 = plan.get<"slot_cmp_of_aux">().add(data<sch::UInt8>{1u}); + SAW_EXPECT(not eo4.is_error(), "slot_cmp_of_aux grows"); + auto eo5 = plan.get<"aux_names">().add(data<sch::String>{std::string{"info"}}); + SAW_EXPECT(not eo5.is_error(), "aux_names grows"); + + auto eo6 = gl.get<"feasible">().add(std::move(plan)); + SAW_EXPECT(not eo6.is_error(), "feasible grows"); + } + + auto eo_text = encode_message<sch::MsgHelloWebGl2>(hello); + SAW_EXPECT(not eo_text.is_error(), "A complete capability must encode"); + + // The header has to be readable out of the biggest message we have. + auto eo_hdr = peek_header(eo_text.get_value()); + SAW_EXPECT(not eo_hdr.is_error(), "The header must survive a large payload"); + SAW_EXPECT(header_type(eo_hdr.get_value()) == msg::hello, "Header reports hello"); + + // Routing pass: the common fields alone are enough to learn the kind, and + // the webgl2 section is skipped because it is not named here. + auto eo_route = decode_message<sch::MsgHello>(eo_text.get_value()); + SAW_EXPECT(not eo_route.is_error(), "The routing decode must succeed"); + SAW_EXPECT(eo_route.get_value().get<"capability">().get<"kind">().stl_view() + == wkind::webgl2, "The routing decode must report the kind"); + + auto eo_back = decode_message<sch::MsgHelloWebGl2>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "A complete capability must decode"); + auto& back = eo_back.get_value().get<"capability">(); + + SAW_EXPECT(back.get<"kind">().stl_view() == wkind::webgl2, "kind survives"); + SAW_EXPECT(back.get<"webgl2">().get<"limits">().get<"max_texture_size">().get() + == 16384u, "Deeply nested integers survive"); + SAW_EXPECT(back.get<"webgl2">().get<"features">().get<"color_buffer_float">().get(), + "Booleans survive"); + SAW_EXPECT(back.get<"webgl2">().get<"budget">().get<"probe_trials">().get() == 14u, + "Budget survives"); + + auto& feasible = back.get<"webgl2">().get<"feasible">(); + SAW_EXPECT(feasible.meta().flat_at(0u).get() == 1u, "One plan came back"); + auto& plan_back = feasible.at({{0u}}); + SAW_EXPECT(plan_back.get<"q">().get() == 9u, "Plan Q survives"); + SAW_EXPECT(plan_back.get<"layout">().stl_view() == "plain2d", "Plan layout survives"); + SAW_EXPECT(plan_back.get<"slot_tex_of_q">().meta().flat_at(0u).get() == 9u, + "The slot table keeps its length"); + SAW_EXPECT(plan_back.get<"slot_tex_of_q">().at({{8u}}).get() == 2u, + "Direction 8 lives in texture 2"); + SAW_EXPECT(plan_back.get<"slot_cmp_of_q">().at({{8u}}).get() == 0u, + "Direction 8 lives in channel 0"); + SAW_EXPECT(plan_back.get<"aux_names">().at({{0u}}).stl_view() == "info", + "Arrays of strings survive"); +} + +/** + * A worker of a kind we do not know about still has to be readable far enough + * to find out that we do not know about it. This is the envelope earning its + * keep. + */ +SAW_TEST("Wire / an unknown worker kind is still routable"){ + const char* text = + R"({"type":"hello","worker_id":"w-x","seq":0,)" + R"("capability":{"worker_id":"w-x","kind":"webgpu","schema_version":"1",)" + R"("reported_at_ms":1,"user_agent":"a","webgpu":{"whatever":true}}})"; + + auto eo_hdr = peek_header(text); + SAW_EXPECT(not eo_hdr.is_error(), + "An unknown backend section must not stop the header decoding"); + SAW_EXPECT(header_type(eo_hdr.get_value()) == msg::hello, "Still a hello"); + SAW_EXPECT(header_worker_id(eo_hdr.get_value()) == "w-x", "Still identifiable"); + + // The routing decode succeeds and names the kind, which is exactly what it + // is for: an unusable worker is still an identifiable worker. + auto eo_route = decode_message<sch::MsgHello>(text); + SAW_EXPECT(not eo_route.is_error(), + "The common fields of an unknown kind must still decode"); + SAW_EXPECT(eo_route.get_value().get<"capability">().get<"kind">().stl_view() + == "webgpu", "The orchestrator learns which kind it cannot use"); + + // Decoding it as a webgl2 hello must fail, because "webgl2" is absent -- + // that is the orchestrator learning it cannot use this worker, which is a + // different and better outcome than a zeroed capability. + auto eo_full = decode_message<sch::MsgHelloWebGl2>(text); + SAW_EXPECT(eo_full.is_error(), + "A webgl2 decode of a webgpu worker must fail rather than zero fill"); +} +/** + * Native workers are a worker kind, not a test fixture, so the same routing has + * to work for them -- and crucially a native hello must not be obliged to carry + * a texture budget it has no way to measure. + */ +SAW_TEST("Wire / a native worker registers without a webgl section"){ + auto hello = make_message<sch::MsgHelloNative>(msg::hello, "n-1", 0u); + + auto& cap = hello.get<"capability">(); + cap.get<"worker_id">().set(std::string{"n-1"}); + cap.get<"kind">().set(std::string{wkind::native}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(1700000000000u); + cap.get<"user_agent">().set(std::string{"kel-lbm-worker-native"}); + + auto& nat = cap.get<"native">(); + nat.get<"threads">().set(16u); + nat.get<"memory_bytes">().set(34359738368u); + nat.get<"cpu">().set(std::string{"x86_64"}); + nat.get<"has_sycl">().set(false); + + auto eo_text = encode_message<sch::MsgHelloNative>(hello); + SAW_EXPECT(not eo_text.is_error(), "A native hello must encode"); + + // Routes the same way a browser does. + auto eo_route = decode_message<sch::MsgHello>(eo_text.get_value()); + SAW_EXPECT(not eo_route.is_error(), "A native hello must route"); + SAW_EXPECT(eo_route.get_value().get<"capability">().get<"kind">().stl_view() + == wkind::native, "Routing reports native"); + + auto eo_back = decode_message<sch::MsgHelloNative>(eo_text.get_value()); + SAW_EXPECT(not eo_back.is_error(), "A native hello must decode"); + SAW_EXPECT(eo_back.get_value().get<"capability">().get<"native">() + .get<"threads">().get() == 16u, "Thread count survives"); + + // And the point of the restructure: this text has no webgl2 section at all, + // so a decode that demands one has to fail rather than zero fill. + auto eo_wrong = decode_message<sch::MsgHelloWebGl2>(eo_text.get_value()); + SAW_EXPECT(eo_wrong.is_error(), + "A native worker must not decode as a webgl2 one"); +} +} diff --git a/modules/sycl/.nix/derivation.nix b/modules/sycl/.nix/derivation.nix index bfcb541..b937cea 100644 --- a/modules/sycl/.nix/derivation.nix +++ b/modules/sycl/.nix/derivation.nix @@ -7,7 +7,7 @@ , forstio , kel , adaptive-cpp -# Names the build so the ROCm and non-ROCm variants land in distinct outputs. +# Names the build so the host, ROCm and CUDA variants land in distinct outputs. , variant ? "" }: diff --git a/modules/webgl/.nix/derivation.nix b/modules/webgl/.nix/derivation.nix new file mode 100644 index 0000000..3654615 --- /dev/null +++ b/modules/webgl/.nix/derivation.nix @@ -0,0 +1,58 @@ +{ lib +, stdenv +, scons +, clang-tools +, emscripten +, python3 +, nodejs +, pname +, version +, kel +}: + +stdenv.mkDerivation { + pname = "${pname}-webgl"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + emscripten + python3 + nodejs + ]; + + # Deliberately empty. em++ is not a nix cc-wrapper, so it never reads + # NIX_CFLAGS_COMPILE and cannot consume buildInputs the way core and sycl do. + # Include roots are handed to scons explicitly below, which also has the + # virtue of making the dependency visible rather than ambient. + buildInputs = [ ]; + + sconsFlags = [ + "kel_include=${kel.lbm.core.dev}/include:${kel.lbm.swarm.dev}/include" + ]; + + # Same class of problem as the `export HOME="$TMPDIR"` AdaptiveCpp hack in + # modules/sycl/.nix/derivation.nix: a toolchain that insists on a writable + # cache inside a read-only build. + # + # emscripten's wrappers source locate_cache.sh, which points EM_CACHE at a + # /tmp path. In the sandbox that is writable but empty, so emcc would + # rebuild the entire sysroot for every target. Seed it from the cache the + # emscripten derivation already precompiled (~81MB, with sysroot and + # sysroot_install.stamp) and make it writable. + preBuild = '' + export HOME="$TMPDIR" + export EM_CACHE="$TMPDIR/em_cache" + cp -r ${emscripten}/share/emscripten/cache "$EM_CACHE" + chmod -R u+w "$EM_CACHE" + ''; + + # Nothing here can run without a browser; the probe only has to link. + doCheck = false; + + preferLocalBuild = true; + + outputs = [ "out" "dev" ]; +} diff --git a/modules/webgl/SConstruct b/modules/webgl/SConstruct new file mode 100644 index 0000000..729c92f --- /dev/null +++ b/modules/webgl/SConstruct @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +# em++ emits a .js loader plus a .wasm sidecar, but SCons only learns about the +# target it was asked for, so the .wasm is invisible to the dependency graph: +# `scons -c` leaves it behind and an incremental build can reuse a stale one. +# +# Program's emitter is an EmitterProxy rather than a dict, so add_emitter() is +# not available on it. Declaring the sidecar explicitly is the supported route: +# SideEffect keeps parallel builds from racing on it, Clean makes -c remove it. +def em_program(env, target, objects): + js = env.Program(target, objects) + wasm = env.File(str(js[0])[:-3] + '.wasm') + env.SideEffect(wasm, js) + env.Clean(js, wasm) + return [js[0], wasm] + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +# em++ is not a nix cc-wrapper, so it never sees NIX_CFLAGS_COMPILE and cannot +# pick up buildInputs the way every other module here does. The derivation +# passes the include roots explicitly instead; see .nix/derivation.nix. +env_vars.Add('kel_include', + help='Colon separated include roots for kel-lbm headers', + default='' +) + +env=Environment(ENV=os.environ, variables=env_vars, + CPPPATH=[p for p in ARGUMENTS.get('kel_include', '').split(':') if p], + CPPDEFINES=[], + CXX = 'em++', + CC = 'emcc', + AR = 'emar', + RANLIB = 'emranlib', + LINK = 'em++', + CXXFLAGS=[ + '-std=c++20', + '-O2', + '-Wall', + '-Wextra', + # kel-lbm's string_literal provides operator""_sl as a string literal + # operator template, which is a GNU extension the library leans on + # deliberately. Without this it warns once per translation unit from a + # core header and drowns out warnings that are actually ours. + '-Wno-gnu-string-literal-operator-template' + ], + LINKFLAGS=[ + # WebGL2 only, and the GLES3 entry points resolved through + # getProcAddress so extension functions work. + '-sFULL_ES3=1', + '-sWASM=1', + '-sMODULARIZE=1', + '-sENVIRONMENT=web', + '-sALLOW_MEMORY_GROWTH=1', + '-sMIN_WEBGL_VERSION=2', + '-sMAX_WEBGL_VERSION=2' + ], + LIBS=[], + PROGSUFFIX='.js' +); +env.__class__.add_source_files = add_kel_source_files +env.__class__.em_program = em_program + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('c++/SConscript') + +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/webgl/c++/SConscript b/modules/webgl/c++/SConscript new file mode 100644 index 0000000..d75d7e0 --- /dev/null +++ b/modules/webgl/c++/SConscript @@ -0,0 +1,31 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +webgl_env = env.Clone(); + +webgl_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +webgl_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += webgl_env.sources; +env.headers += webgl_env.headers; + +# The module is headers plus probe.cpp. toolchain_probe.cpp is the smoke test +# target: it proves em++ can compile against the core and swarm headers and +# that both the .js and the .wasm land where SCons expects them. +objects = [] +webgl_env.add_source_files(objects, webgl_env.sources, shared=False); +env.probe = webgl_env.em_program('#bin/kel_lbm_webgl_probe', [objects]); + +env.targets += [env.probe]; + +env.Install('$prefix/include/kel/lbm/webgl/', webgl_env.headers); +# Both the loader and the sidecar; the .js is useless without the .wasm. +env.Install('$prefix/share/kel-lbm/webgl/', env.probe); diff --git a/modules/webgl/c++/context.hpp b/modules/webgl/c++/context.hpp new file mode 100644 index 0000000..2f04c16 --- /dev/null +++ b/modules/webgl/c++/context.hpp @@ -0,0 +1,112 @@ +#pragma once + +#include <emscripten/html5.h> +#include <emscripten/html5_webgl.h> + +#include <GLES3/gl3.h> + +#include <string> + +/** + * Owning a WebGL2 context. + * + * The probe deliberately runs on its own throwaway context rather than the one + * the simulation will use. Measuring capacity means allocating until something + * breaks, and when a browser hits GPU exhaustion it frequently answers by + * killing the context rather than returning an error. Losing a context created + * for the purpose costs nothing; losing the one holding the lattice costs the + * run. + */ +namespace kel { +namespace lbm { +namespace webgl { + +class context { +public: + context() = default; + + context(const context &) = delete; + context &operator=(const context &) = delete; + + ~context(){ destroy(); } + + /** + * `target` is a CSS selector for the canvas, e.g. "#probe". In a worker + * this needs an OffscreenCanvas transferred from the main thread and the + * build flag to match; see the module's SConstruct. + */ + bool create(const std::string &target){ + destroy(); + + EmscriptenWebGLContextAttributes attrs; + emscripten_webgl_init_context_attributes(&attrs); + + attrs.majorVersion = 2; + attrs.minorVersion = 0; + + // None of this is drawn to. Turning the extras off keeps the context + // cheap and avoids the compositor holding buffers we never read. + attrs.alpha = EM_FALSE; + attrs.depth = EM_FALSE; + attrs.stencil = EM_FALSE; + attrs.antialias = EM_FALSE; + + // Accept a software context rather than refusing one. A modest device + // should still be able to join: what it reports here is a memory + // budget, and that budget is real even when the renderer behind it is + // slow. The cost is that capacity alone cannot distinguish a software + // context from a fast one -- that is a throughput question, which is + // what the perf section of the capability is reserved for. + attrs.failIfMajorPerformanceCaveat = EM_FALSE; + + handle_ = emscripten_webgl_create_context(target.c_str(), &attrs); + if(handle_ <= 0){ + handle_ = 0; + return false; + } + + if(emscripten_webgl_make_context_current(handle_) != EMSCRIPTEN_RESULT_SUCCESS){ + destroy(); + return false; + } + + return true; + } + + bool valid() const { return handle_ > 0; } + + bool lost() const { + return handle_ > 0 and emscripten_is_webgl_context_lost(handle_); + } + + /** + * Extensions have to be asked for by name before the entry points work, + * even where the underlying GLES symbol already exists. + */ + bool enable_extension(const char *name){ + if(handle_ <= 0){ + return false; + } + return emscripten_webgl_enable_extension(handle_, name); + } + + bool make_current(){ + return handle_ > 0 and + emscripten_webgl_make_context_current(handle_) + == EMSCRIPTEN_RESULT_SUCCESS; + } + + void destroy(){ + if(handle_ > 0){ + emscripten_webgl_destroy_context(handle_); + handle_ = 0; + } + } + +private: + EMSCRIPTEN_WEBGL_CONTEXT_HANDLE handle_ = 0; +}; + +} // namespace webgl +} // namespace lbm +} // namespace kel diff --git a/modules/webgl/c++/probe.hpp b/modules/webgl/c++/probe.hpp new file mode 100644 index 0000000..5dbc33c --- /dev/null +++ b/modules/webgl/c++/probe.hpp @@ -0,0 +1,321 @@ +#pragma once + +#include "context.hpp" + +#include <kel/lbm/abstract/data.hpp> +#include <kel/lbm/swarm/capability.hpp> +#include <kel/lbm/swarm/capacity.hpp> +#include <kel/lbm/swarm/probe_search.hpp> + +#include <cstdint> +#include <string> +#include <vector> + +/** + * Measuring what a browser will actually give you. + * + * Split in two on purpose. The limits and extensions are a handful of queries; + * the budget is a measurement, because WebGL2 has no capacity query and the + * limits it does report overstate the truth by around a hundredfold on a + * typical desktop. + * + * The strategy for that measurement is not here -- it is swarm's + * allocation_search, which is ordinary arithmetic and is unit tested. What + * lives here is only the part that genuinely needs a GPU: allocate, check, + * destroy. Everything in this header is unreachable without a browser, so the + * rule is that it stays thin enough to audit by reading. + * + * Header only, like core and swarm, so a wasm consumer needs no link wiring -- + * which matters because em++ cannot consume nix buildInputs the way the native + * builds can. + */ +namespace kel { +namespace lbm { +namespace webgl { + +/// How a single allocation attempt ended. Not a bool, because "the context +/// died" has to be distinguished from "that was too big". +enum class trial_result { + ok, + /// Rejected, by a GL error or an incomplete framebuffer. + too_big, + /// The context went away. Everything after this is meaningless. + context_lost, +}; + +struct probe_config { + /// CSS selector for the throwaway canvas. + std::string canvas = "#kel-probe"; + + /// Distribution planes to allocate per attempt. Three is D2Q9. + uint32_t planes = 3u; + + /// Both ping pong copies, since the real thing needs both resident. + uint32_t states = 2u; + + /** + * Leaves room for the canvas, the compositor and other tabs. A guess, + * and flagged as one: it wants measuring against a real run on real + * hardware before it is trusted. + */ + float safety_factor = 0.6f; +}; + +namespace impl { + +inline uint32_t get_int(GLenum name){ + GLint value = 0; + glGetIntegerv(name, &value); + return (value > 0) ? static_cast<uint32_t>(value) : 0u; +} + +/// Clears any error already pending, so the next check reports our own. +inline void drain_errors(){ + for(int i = 0; i < 16; ++i){ + if(glGetError() == GL_NO_ERROR){ + return; + } + } +} + +} // namespace impl + +/// Fills the queryable limits. Requires a current context. +inline void query_limits(data<sch::GlLimits> &out){ + out.get<"max_texture_size">().set(impl::get_int(GL_MAX_TEXTURE_SIZE)); + out.get<"max_3d_texture_size">().set(impl::get_int(GL_MAX_3D_TEXTURE_SIZE)); + out.get<"max_array_texture_layers">().set( + impl::get_int(GL_MAX_ARRAY_TEXTURE_LAYERS)); + out.get<"max_draw_buffers">().set(impl::get_int(GL_MAX_DRAW_BUFFERS)); + out.get<"max_color_attachments">().set(impl::get_int(GL_MAX_COLOR_ATTACHMENTS)); + out.get<"max_texture_image_units">().set( + impl::get_int(GL_MAX_TEXTURE_IMAGE_UNITS)); + out.get<"max_combined_texture_image_units">().set( + impl::get_int(GL_MAX_COMBINED_TEXTURE_IMAGE_UNITS)); + out.get<"max_renderbuffer_size">().set(impl::get_int(GL_MAX_RENDERBUFFER_SIZE)); + out.get<"max_fragment_uniform_vectors">().set( + impl::get_int(GL_MAX_FRAGMENT_UNIFORM_VECTORS)); + + GLint dims[2] = {0, 0}; + glGetIntegerv(GL_MAX_VIEWPORT_DIMS, dims); + out.get<"max_viewport_dims">().at({{0u}}).set( + (dims[0] > 0) ? static_cast<uint32_t>(dims[0]) : 0u); + out.get<"max_viewport_dims">().at({{1u}}).set( + (dims[1] > 0) ? static_cast<uint32_t>(dims[1]) : 0u); +} + +/// Fills the extension and context properties. Requires a current context. +inline void query_features(context &ctx, data<sch::GlFeatures> &out){ + out.get<"webgl2">().set(ctx.valid()); + + // Whether this is running off an OffscreenCanvas, which is what a worker + // thread needs in order to have a context at all. +#if defined(__EMSCRIPTEN_PTHREADS__) + out.get<"offscreen_canvas">().set(true); +#else + out.get<"offscreen_canvas">().set(false); +#endif + + // The hard gate. Without it, float textures are not colour renderable. + out.get<"color_buffer_float">().set( + ctx.enable_extension("EXT_color_buffer_float")); + out.get<"color_buffer_half_float">().set( + ctx.enable_extension("EXT_color_buffer_half_float")); + + // Not needed by the solver, which fetches by texel rather than sampling. + // Reported because a smooth visualisation would want it. + out.get<"texture_float_linear">().set( + ctx.enable_extension("OES_texture_float_linear")); + + // Will matter for immersed boundary force spreading, which accumulates by + // blending. Nothing uses it yet. + out.get<"float_blend">().set(ctx.enable_extension("EXT_float_blend")); + + out.get<"parallel_shader_compile">().set( + ctx.enable_extension("KHR_parallel_shader_compile")); + out.get<"disjoint_timer_query">().set( + ctx.enable_extension("EXT_disjoint_timer_query_webgl2")); +} + +/** + * One allocation attempt at `nx` by `ny`. + * + * Does the full sequence rather than just asking for memory: immutable storage, + * framebuffer completeness, and then a real clear plus a one texel readback. + * The last part is not ceremony -- a driver is entitled to defer the actual + * allocation until first use, so an attempt that stops at glTexStorage2D can + * report success against memory that does not exist. + */ +inline trial_result try_allocate(context &ctx, const probe_config &cfg, + uint32_t nx, uint32_t ny){ + if(nx == 0u or ny == 0u){ + return trial_result::too_big; + } + + impl::drain_errors(); + + const uint32_t count = cfg.planes * cfg.states; + + std::vector<GLuint> textures(count, 0u); + GLuint fbo = 0u; + + // Every exit path goes through here. A probe that leaks on failure would + // poison the attempts that follow it, which is precisely the regime where + // failures are expected. + auto cleanup = [&](){ + if(fbo != 0u){ + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glDeleteFramebuffers(1, &fbo); + } + for(GLuint tex : textures){ + if(tex != 0u){ + glDeleteTextures(1, &tex); + } + } + glFlush(); + }; + + glGenTextures(static_cast<GLsizei>(count), textures.data()); + + for(uint32_t i = 0u; i < count; ++i){ + glBindTexture(GL_TEXTURE_2D, textures[i]); + + // Immutable storage. The mutable glTexImage2D path lets a driver defer + // the decision, so it can report success for memory it has not + // committed to. + glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA32F, + static_cast<GLsizei>(nx), static_cast<GLsizei>(ny)); + + if(glGetError() != GL_NO_ERROR){ + cleanup(); + return ctx.lost() ? trial_result::context_lost : trial_result::too_big; + } + + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + + // Attach one state's worth, which is what a real pass renders into. + glGenFramebuffers(1, &fbo); + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + + std::vector<GLenum> attachments; + attachments.reserve(cfg.planes); + for(uint32_t i = 0u; i < cfg.planes; ++i){ + glFramebufferTexture2D(GL_FRAMEBUFFER, + GL_COLOR_ATTACHMENT0 + i, + GL_TEXTURE_2D, textures[i], 0); + attachments.push_back(GL_COLOR_ATTACHMENT0 + i); + } + glDrawBuffers(static_cast<GLsizei>(attachments.size()), attachments.data()); + + if(glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE){ + cleanup(); + return ctx.lost() ? trial_result::context_lost : trial_result::too_big; + } + + // Force the driver to commit. Clearing touches every attachment, and the + // readback blocks until it has actually happened -- without this the whole + // sequence can succeed against memory that was never really allocated. + glViewport(0, 0, static_cast<GLsizei>(nx), static_cast<GLsizei>(ny)); + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + + float pixel[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + glReadBuffer(GL_COLOR_ATTACHMENT0); + glReadPixels(0, 0, 1, 1, GL_RGBA, GL_FLOAT, pixel); + + const GLenum err = glGetError(); + const bool was_lost = ctx.lost(); + + cleanup(); + + if(was_lost){ + return trial_result::context_lost; + } + if(err != GL_NO_ERROR){ + return trial_result::too_big; + } + + return trial_result::ok; +} + +/** + * Runs the whole probe on a throwaway context and fills the capability. + * + * Returns false if a WebGL2 context could not be had at all, or if + * EXT_color_buffer_float is missing -- without it a float texture is not + * colour renderable and there is no solver on this device, so there is nothing + * worth measuring. The capability is still filled in either case: an unusable + * worker is better reported than silent. + */ +inline bool probe_webgl2(const probe_config &cfg, data<sch::WebGl2Capability> &out){ + context ctx; + if(not ctx.create(cfg.canvas)){ + return false; + } + + query_limits(out.get<"limits">()); + query_features(ctx, out.get<"features">()); + + if(not out.get<"features">().get<"color_buffer_float">().get()){ + return false; + } + + const uint64_t max_tex = out.get<"limits">().get<"max_texture_size">().get(); + + const uint64_t bytes_per_cell = + swarm::packing_math::bytes_per_cell(9u, cfg.states, 1u, true); + + swarm::allocation_search::config scfg; + scfg.ceiling_cells = + swarm::capacity_math::texture_bound_cells_2d(max_tex, 0u); + + swarm::allocation_search search{scfg}; + + bool context_lost = false; + uint32_t trials = 0u; + + while(not search.done()){ + // Square-ish, so neither edge runs into max_texture_size before the + // area runs into memory. + const auto dom = swarm::capacity_math::fit_domain_2d( + search.candidate(), 1u, 1u, max_tex, 0u); + + const trial_result result = try_allocate( + ctx, cfg, static_cast<uint32_t>(dom.nx), + static_cast<uint32_t>(dom.ny)); + + ++trials; + + if(result == trial_result::context_lost){ + // Browsers often answer GPU exhaustion by killing the context + // rather than returning an error. Everything after this would be + // meaningless, so stop and keep what was already measured. + context_lost = true; + break; + } + + search.report(result == trial_result::ok); + } + + const uint64_t best_bytes = search.best_cells() * bytes_per_cell; + + auto &budget = out.get<"budget">(); + budget.get<"probed_bytes_max">().set(best_bytes); + budget.get<"probed_bytes_safe">().set( + static_cast<uint64_t>(static_cast<double>(best_bytes) * + static_cast<double>(cfg.safety_factor))); + budget.get<"probe_trials">().set(trials); + budget.get<"context_lost_during_probe">().set(context_lost); + budget.get<"probe_hit_texture_size_bound">().set(search.hit_ceiling()); + budget.get<"safety_factor">().set(cfg.safety_factor); + + return not context_lost and search.best_cells() > 0u; +} + +} // namespace webgl +} // namespace lbm +} // namespace kel diff --git a/modules/webgl/c++/toolchain_probe.cpp b/modules/webgl/c++/toolchain_probe.cpp new file mode 100644 index 0000000..f9e3907 --- /dev/null +++ b/modules/webgl/c++/toolchain_probe.cpp @@ -0,0 +1,97 @@ +/** + * The webgl module's smoke target. + * + * Nothing here can be meaningfully run without a browser, so its job is to make + * the compiler and linker check what a check phase cannot: + * + * 1. em++ finds the kel-lbm core and swarm headers. It is not a nix + * cc-wrapper, so it never reads NIX_CFLAGS_COMPILE and cannot pick up + * buildInputs; the include roots arrive through the kel_include scons + * variable instead. + * 2. The core headers are clean enough for a wasm target. descriptor.hpp is + * the narrow include the worker uses -- deliberately not + * <kel/lbm/lbm.hpp>, which drags in util.hpp's <sys/ioctl.h>, the vtk and + * csv writers and <iostream>. + * 3. Every GL and emscripten entry point the probe calls actually resolves. + * Emscripten sets ERROR_ON_UNDEFINED_SYMBOLS by default, so linking this + * is a real check on probe.cpp rather than a formality. + * 4. The chain the browser worker depends on composes end to end: probe a + * capability, put it in a hello, encode it to the text that goes on the + * data channel. + * 5. Both the .js loader and the .wasm sidecar are produced. + * + * The descriptor tables are consumed through a static_assert rather than merely + * included, so the compiler is forced to instantiate df_info and this cannot + * silently degrade into "the header parsed". + */ + +#include "probe.hpp" + +#include <kel/lbm/descriptor.hpp> +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <cstdint> +#include <cstdio> +#include <string> + +namespace { +// Qualified rather than aliased: a local `sch` would collide with kel::lbm::sch +// the moment main pulls kel::lbm into scope. +using Desc = kel::lbm::sch::D2Q9; +using dfi = kel::lbm::df_info<kel::lbm::sch::Float32, Desc>; + +static_assert(Desc::D == 2u, "D2Q9 has to be two dimensional"); +static_assert(Desc::Q == 9u, "D2Q9 has to carry nine directions"); + +/** + * Opposites have to pair up. This is the cheapest property that actually + * touches the constexpr tables, which is the point -- it proves instantiation + * rather than parsing. + */ +constexpr bool opposites_pair(){ + for(uint64_t i = 0u; i < Desc::Q; ++i){ + if(dfi::opposite_index[dfi::opposite_index[i]] != i){ + return false; + } + } + return true; +} + +static_assert(opposites_pair(), "opposite_index has to be an involution"); +} + +int main(){ + using namespace kel::lbm; + + webgl::probe_config cfg; + + // Without a canvas this returns false immediately, which is the correct + // behaviour and is all that can be checked outside a browser. The call is + // here so the signatures and the capability plumbing are compiled against. + data<sch::WebGl2Capability> gl{}; + const bool usable = webgl::probe_webgl2(cfg, gl); + + // The chain a real worker walks: capability into a hello, hello onto the + // wire. If this compiles and links, step six has no surprises waiting. + auto hello = swarm::make_message<sch::MsgHelloWebGl2>( + msg::hello, "probe", 0u); + + auto &cap = hello.get<"capability">(); + cap.get<"worker_id">().set(std::string{"probe"}); + cap.get<"kind">().set(std::string{wkind::webgl2}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(0u); + cap.get<"user_agent">().set(std::string{"kel-lbm-webgl-probe"}); + cap.get<"webgl2">() = gl; + + auto eo_text = swarm::encode_message<sch::MsgHelloWebGl2>(hello); + if(eo_text.is_error()){ + return 1; + } + + std::printf("usable=%d hello=%zu bytes\n", + usable ? 1 : 0, eo_text.get_value().size()); + + return 0; +} diff --git a/poc/default.nix b/poc/default.nix index e2c41bd..ceeec4b 100644 --- a/poc/default.nix +++ b/poc/default.nix @@ -1,6 +1,12 @@ { pkgs ? import <nixpkgs> {} +, stdenv +, pname +, version +, kel }: { - web_swarm = import ./web_swarm/default.nix {}; + web_swarm = import ./web_swarm/default.nix { + inherit pkgs stdenv pname version kel; + }; } diff --git a/poc/web_swarm/.nix/derivation.nix b/poc/web_swarm/.nix/derivation.nix new file mode 100644 index 0000000..31f1e2b --- /dev/null +++ b/poc/web_swarm/.nix/derivation.nix @@ -0,0 +1,52 @@ +{ lib +, stdenv +, scons +, clang-tools +, libdatachannel +, pname +, version +, kel +}: + +stdenv.mkDerivation { + pname = "${pname}-web-swarm"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + # Ordinary native derivations, so unlike the webgl module these get their + # include paths from nix the usual way. + # + # libdatachannel ships CMake files but no pkg-config, hence the plain + # LIBS=['datachannel'] in the SConstruct rather than a pkg-config lookup. + # Note there is no forstio here: libdatachannel owns the threads and the + # event handling, and forstio has no WebSocket or WebRTC to offer. + buildInputs = [ + libdatachannel + kel.lbm.core + kel.lbm.swarm + ]; + + # No checkPhase, and that is a limitation rather than a choice. The nix build + # sandbox runs in a network namespace holding only a loopback interface, and + # ICE finds no usable candidate there -- libdatachannel reports "Failed to + # gather local ICE candidates" and negotiation never completes. Verified: the + # same two binaries negotiate and register fine outside the sandbox. + # + # The end to end check therefore ships as an executable instead, so it is one + # command rather than a procedure: + # + # nix-build -A poc.web_swarm.native + # ./result/bin/kel_lbm_web_swarm_check + # + # The protocol itself is covered natively and in-sandbox by kel.lbm.swarm. + doCheck = false; + + preferLocalBuild = true; + + outputs = [ "out" ]; +} diff --git a/poc/web_swarm/SConscript b/poc/web_swarm/SConscript deleted file mode 100644 index 64d0b79..0000000 --- a/poc/web_swarm/SConscript +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/false - -import os -import os.path -import glob - - -Import('env') - -dir_path = Dir('.').abspath - -# Environment for base library -examples_env = env.Clone(); -examples_env['CXX'] = 'syclcc-clang'; -examples_env['CXXFLAGS'] += ['-O3']; - -examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) -examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) - -env.sources += examples_env.sources; -env.headers += examples_env.headers; - -# Cavity2D -examples_objects = []; -examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); -examples_env.poiseulle_2d_gpu = examples_env.Program('#bin/kel_lbm_web_swarm', [examples_objects]); - -# Set Alias -env.examples = [ - examples_env.poiseulle_2d_gpu -]; -env.Alias('examples', env.examples); -env.targets += ['examples']; -env.Install('$prefix/bin/', env.examples); diff --git a/poc/web_swarm/SConstruct b/poc/web_swarm/SConstruct index 0611b67..9d43609 100644 --- a/poc/web_swarm/SConstruct +++ b/poc/web_swarm/SConstruct @@ -43,23 +43,20 @@ env_vars.Add('prefix', validator=isAbsolutePath ) -env_vars.Add('build_examples', - help='If examples should be built', - default="true" -) - +# No forstio. The orchestrator is the one native binary here that does not use +# it: libdatachannel brings its own threads and event handling, and forstio has +# no WebSocket or WebRTC to contribute. env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], - CPPDEFINES=['SAW_UNIX'], + CPPDEFINES=[], CXXFLAGS=[ '-std=c++20', '-g', + '-O2', '-Wall', '-Wextra' ], LIBS=[ - 'forstio-core', - 'forstio-async', - 'forstio-io' + 'datachannel' ] ); env.__class__.add_source_files = add_kel_source_files @@ -71,11 +68,16 @@ env.sources = []; env.headers = []; env.targets = []; +env.check_script = File('#check.sh'); + Export('env') -SConscript('SConscript') +SConscript('orchestrator/c++/SConscript') +SConscript('worker_native/c++/SConscript') env.Alias('cdb', env.cdb); env.Alias('all', [env.targets]); env.Default('all'); +env.InstallAs('$prefix/bin/kel_lbm_web_swarm_check', env.check_script); + env.Alias('install', '$prefix') diff --git a/poc/web_swarm/check.sh b/poc/web_swarm/check.sh new file mode 100755 index 0000000..4cf6b87 --- /dev/null +++ b/poc/web_swarm/check.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# End to end check: does a native worker register with the orchestrator over a +# real WebRTC data channel? +# +# This is not a nix checkPhase, and that is not an oversight. The nix build +# sandbox runs in a network namespace holding nothing but a loopback interface, +# and ICE finds no usable candidate there -- libdatachannel reports "Failed to +# gather local ICE candidates" and negotiation never completes. The binaries are +# fine; there is simply nothing to gather. So the check lives here and is run +# outside the sandbox: +# +# nix-build -A poc.web_swarm.native +# ./result/bin/kel_lbm_web_swarm_check +# +# Exits 0 on success, non-zero with both logs printed otherwise. + +set -u + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +orchestrator="$here/kel_lbm_orchestrator" +worker="$here/kel_lbm_worker_native" + +for binary in "$orchestrator" "$worker"; do + if [ ! -x "$binary" ]; then + echo "missing executable: $binary" >&2 + exit 1 + fi +done + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT + +orch_log="$workdir/orch.log" +worker_log="$workdir/worker.log" + +# Port 0 asks the OS for a free one, so the port is read back rather than +# assumed -- otherwise two runs on one machine collide. +# Fast heartbeat so three beats land well inside the worker timeout. +"$orchestrator" --port 0 --heartbeat 300 --monitor 0 > "$orch_log" 2>&1 & +orch_pid=$! +trap 'kill $orch_pid 2>/dev/null; rm -rf "$workdir"' EXIT + +port="" +for _ in $(seq 1 100); do + port="$(sed -n 's|.*ws://0\.0\.0\.0:\([0-9]\+\).*|\1|p' "$orch_log")" + [ -n "$port" ] && break + if ! kill -0 $orch_pid 2>/dev/null; then + echo "orchestrator exited before reporting a port:" >&2 + cat "$orch_log" >&2 + exit 1 + fi + sleep 0.1 +done + +if [ -z "$port" ]; then + echo "orchestrator never reported a port:" >&2 + cat "$orch_log" >&2 + exit 1 +fi + +echo "orchestrator listening on $port" + +"$worker" \ + --url "ws://127.0.0.1:$port/" \ + --id check-native \ + --once \ + --pongs 3 \ + --timeout 20000 > "$worker_log" 2>&1 +worker_rc=$? + +kill $orch_pid 2>/dev/null +wait $orch_pid 2>/dev/null + +fail(){ + echo "FAIL: $1" >&2 + echo "--- orchestrator ---" >&2; cat "$orch_log" >&2 + echo "--- worker ---" >&2; cat "$worker_log" >&2 + exit 1 +} + +[ $worker_rc -eq 0 ] || fail "worker exited $worker_rc" + +# Both sides have to agree the channel came up, not just the worker. +grep -q "channel open" "$orch_log" \ + || fail "orchestrator never saw the control channel open" +grep -q "control channel open" "$worker_log" \ + || fail "worker never saw the control channel open" + +# The hello has to have been accepted, as a native worker, and answered. +grep -q "registered check-native (native)" "$orch_log" \ + || fail "orchestrator never registered the worker as native" +grep -q "<- welcome" "$worker_log" \ + || fail "worker never received its welcome" + +# Liveness, not just registration: the heartbeat has to be running and the +# worker has to be answering it. +grep -q "answered 3 ping(s)" "$worker_log" \ + || fail "worker did not answer three heartbeats" + +echo "PASS: native worker registered over a real data channel" diff --git a/poc/web_swarm/default.nix b/poc/web_swarm/default.nix index 570debf..e9ea108 100644 --- a/poc/web_swarm/default.nix +++ b/poc/web_swarm/default.nix @@ -1,6 +1,20 @@ { pkgs ? import <nixpkgs> {} +, stdenv +, pname +, version +, kel }: { - ui = pkgs.callPackage ./ui/.nix/derivation.nix {}; + # Orchestrator and native worker share a SConstruct and are checked against + # each other, so they are one derivation. + native = pkgs.callPackage ./.nix/derivation.nix { + inherit stdenv pname version kel; + }; + + # The browser worker is separate: different toolchain, and it cannot be + # tested in a check phase because a check phase has no browser. + worker = pkgs.callPackage ./worker/.nix/derivation.nix { + inherit stdenv pname version kel; + }; } diff --git a/poc/web_swarm/orchestrater/c++/main.cpp b/poc/web_swarm/orchestrater/c++/main.cpp deleted file mode 100644 index 891095e..0000000 --- a/poc/web_swarm/orchestrater/c++/main.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include <forstio/codec/args.hpp> - -namespace kel { -namespace sch { -} - -kel::lbm::error_or<void> kmain(int argc, char** argv){ - kel::lbm::data<sch::Args> args; - // Decode the args - { - kel::lbm::data<sch::Args,encode::Args> arge; - auto eov = saw::parse_args<sch::Args>(arge,args); - if(eov.is_error()){ - return eov; - } - } - return kel::lbm::make_void(); -} - -} - -int main(int argc, char** argv){ - auto eov = kel::kmain(argc,argv); - if(eov.is_error()){ - auto& err = eov.get_error(); - } - return 0; -} diff --git a/poc/web_swarm/orchestrator/c++/SConscript b/poc/web_swarm/orchestrator/c++/SConscript new file mode 100644 index 0000000..c4a7608 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/SConscript @@ -0,0 +1,26 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +orch_env = env.Clone(); + +orch_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +orch_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += orch_env.sources; +env.headers += orch_env.headers; + +objects = [] +orch_env.add_source_files(objects, orch_env.sources, shared=False); +env.orchestrator = orch_env.Program('#bin/kel_lbm_orchestrator', [objects]); + +env.targets += [env.orchestrator]; + +env.Install('$prefix/bin/', env.orchestrator); diff --git a/poc/web_swarm/orchestrator/c++/control.cpp b/poc/web_swarm/orchestrator/c++/control.cpp new file mode 100644 index 0000000..6e2d7b7 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/control.cpp @@ -0,0 +1,276 @@ +#include "control.hpp" + +#include <kel/lbm/swarm/interpret.hpp> +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <chrono> +#include <iomanip> +#include <sstream> +#include <utility> + +namespace kel { +namespace lbm { +namespace swarm { + +namespace { + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +} // namespace + +control::control(orchestrator &orch, control_config cfg) + : orch_{orch}, cfg_{std::move(cfg)}, registry_{cfg_.limits} {} + +control::~control(){ + stop_heartbeat(); +} + +void control::on_event(std::function<void(const std::string &)> handler){ + on_event_ = std::move(handler); +} + +void control::emit(const std::string &line){ + if(on_event_){ + on_event_(line); + } +} + +void control::attach(){ + orch_.on_channel_open([this](const std::shared_ptr<session> &sess){ + registry_.add_session(sess->id, now_ms()); + emit("[" + sess->id + "] channel open, awaiting hello"); + }); + + orch_.on_channel_closed([this](const std::shared_ptr<session> &sess){ + auto rec = registry_.remove(sess->id); + if(rec){ + emit("[" + sess->id + "] disconnected (" + + (rec->worker_id.empty() ? "unregistered" : rec->worker_id) + ")"); + } + }); + + orch_.on_message([this](const std::shared_ptr<session> &sess, + const std::string &text){ + handle_message(sess, text); + }); +} + +void control::handle_message(const std::shared_ptr<session> &sess, + const std::string &text){ + auto eo_hdr = peek_header(text); + if(eo_hdr.is_error()){ + emit("[" + sess->id + "] malformed control message, ignoring"); + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + const uint64_t seq = eo_hdr.get_value().get<"seq">().get(); + + if(type == msg::hello){ + handle_hello(sess, text); + return; + } + + if(type == msg::pong){ + auto eo = decode_message<sch::MsgPong>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed pong, ignoring"); + return; + } + registry_.record_pong(sess->id, seq, + eo.get_value().get<"sent_at_ms">().get(), + eo.get_value().get<"live">(), now_ms()); + return; + } + + if(type == msg::ready){ + auto eo = decode_message<sch::MsgReady>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed ready, ignoring"); + return; + } + registry_.mark_ready(sess->id, eo.get_value().get<"live">(), now_ms()); + emit("[" + sess->id + "] ready"); + return; + } + + if(type == msg::status){ + auto eo = decode_message<sch::MsgStatus>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed status, ignoring"); + return; + } + const auto &live = eo.get_value().get<"live">(); + registry_.update_live(sess->id, live, now_ms()); + + const std::string last_error = live.get<"last_error">().stl_string(); + if(not last_error.empty()){ + emit("[" + sess->id + "] reports error: " + last_error); + } + return; + } + + emit("[" + sess->id + "] unknown message type '" + type + "', ignoring"); +} + +void control::handle_hello(const std::shared_ptr<session> &sess, + const std::string &text){ + // Routing pass: the common capability fields are enough to learn the kind, + // and the backend section is skipped because it is not named here. + auto eo_route = decode_message<sch::MsgHello>(text); + if(eo_route.is_error()){ + emit("[" + sess->id + "] hello missing its common fields, rejecting"); + return; + } + + const auto &common = eo_route.get_value().get<"capability">(); + const std::string kind = common.get<"kind">().stl_string(); + const std::string worker_id = common.get<"worker_id">().stl_string(); + const std::string their_version = common.get<"schema_version">().stl_string(); + + if(their_version != schema_version){ + // Caught deliberately rather than suffered as a decode failure later. + emit("[" + sess->id + "] schema version mismatch: worker says '" + + their_version + "', orchestrator is '" + std::string{schema_version} + + "' -- rejecting"); + return; + } + + uint64_t cells = 0u; + std::string summary; + + if(kind == wkind::webgl2){ + auto eo = decode_message<sch::MsgHelloWebGl2>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] hello claims webgl2 but has no usable " + "webgl2 section, rejecting"); + return; + } + const auto &gl = eo.get_value().get<"capability">().get<"webgl2">(); + cells = webgl2_capacity_cells(gl); + summary = webgl2_summary(gl); + + } else if(kind == wkind::native){ + auto eo = decode_message<sch::MsgHelloNative>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] hello claims native but has no usable " + "native section, rejecting"); + return; + } + const auto &nat = eo.get_value().get<"capability">().get<"native">(); + cells = native_capacity_cells(nat); + summary = native_summary(nat); + + } else { + // Identifiable but unusable, which is a better outcome than a zeroed + // registration and is exactly what the routing decode is for. + emit("[" + sess->id + "] unsupported worker kind '" + kind + + "', rejecting"); + return; + } + + const uint64_t t = now_ms(); + if(not registry_.mark_registered(sess->id, worker_id, kind, cells, + summary, t)){ + return; + } + + auto welcome = make_message<sch::MsgWelcome>(msg::welcome, sess->id, 0u); + welcome.get<"heartbeat_ms">().set(cfg_.heartbeat_ms); + welcome.get<"schema_version">().set(std::string{schema_version}); + + auto eo_out = encode_message<sch::MsgWelcome>(welcome); + if(eo_out.is_error()){ + return; + } + orch_.send(sess, eo_out.get_value()); + + std::ostringstream line; + line << "[" << sess->id << "] registered " << worker_id + << " (" << kind << ") -- " << summary; + emit(line.str()); +} + +void control::start_heartbeat(){ + if(running_.exchange(true)){ + return; + } + ticker_ = std::thread{[this](){ heartbeat_loop(); }}; +} + +void control::stop_heartbeat(){ + if(not running_.exchange(false)){ + return; + } + tick_cv_.notify_all(); + if(ticker_.joinable()){ + ticker_.join(); + } +} + +void control::heartbeat_loop(){ + while(running_.load()){ + { + std::unique_lock<std::mutex> lock{tick_mutex_}; + tick_cv_.wait_for(lock, std::chrono::milliseconds{cfg_.heartbeat_ms}, + [this](){ return not running_.load(); }); + } + if(not running_.load()){ + break; + } + + const uint64_t t = now_ms(); + + // The registry decides; this loop only carries out the decisions. In + // particular nothing is sent while its lock is held. + auto result = registry_.tick(t); + + for(const auto &id : result.newly_stale){ + emit("[" + id + "] stale, no response to the last " + + std::to_string(cfg_.limits.stale_after) + " pings"); + } + + for(const auto &rec : result.dropped){ + emit("[" + rec.session_id + "] dropped after " + + std::to_string(cfg_.limits.drop_after) + " missed pings (" + + (rec.worker_id.empty() ? "unregistered" : rec.worker_id) + ")"); + + // Tear the transport down too, or the session lingers with nothing + // watching it. + if(auto sess = orch_.find_session(rec.session_id)){ + if(sess->dc){ + sess->dc->close(); + } + if(sess->pc){ + sess->pc->close(); + } + } + } + + for(const auto &target : result.to_ping){ + auto sess = orch_.find_session(target.session_id); + if(not sess){ + continue; + } + + auto ping = make_message<sch::MsgPing>(msg::ping, target.session_id, + target.seq); + ping.get<"sent_at_ms">().set(t); + + auto eo = encode_message<sch::MsgPing>(ping); + if(eo.is_error()){ + continue; + } + orch_.send(sess, eo.get_value()); + } + } +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/control.hpp b/poc/web_swarm/orchestrator/c++/control.hpp new file mode 100644 index 0000000..9365ef9 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/control.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "orchestrator.hpp" +#include <kel/lbm/swarm/registry.hpp> + +#include <atomic> +#include <condition_variable> +#include <cstdint> +#include <functional> +#include <mutex> +#include <string> +#include <thread> + +namespace kel { +namespace lbm { +namespace swarm { + +struct control_config { + uint64_t heartbeat_ms = 2000u; + registry::thresholds limits; +}; + +/** + * The control protocol on top of the transport. + * + * Kept apart from `orchestrator` so each can be reasoned about alone: the + * transport knows nothing about hellos, and this knows nothing about ICE. + * + * Owns the heartbeat thread. libdatachannel provides no timers, so liveness is + * a plain ticker -- which is fine, since the work it does per beat is a lock, a + * vector copy and some sends. + */ +class control { +public: + control(orchestrator &orch, control_config cfg); + ~control(); + + control(const control &) = delete; + control &operator=(const control &) = delete; + + /// Installs the handlers on the orchestrator. Call before listen(). + void attach(); + + void start_heartbeat(); + void stop_heartbeat(); + + registry &workers() { return registry_; } + + /// Emitted for anything worth a line in the log. + void on_event(std::function<void(const std::string &)> handler); + +private: + void handle_message(const std::shared_ptr<session> &sess, + const std::string &text); + void handle_hello(const std::shared_ptr<session> &sess, + const std::string &text); + void heartbeat_loop(); + void emit(const std::string &line); + + orchestrator &orch_; + control_config cfg_; + registry registry_; + + std::thread ticker_; + std::mutex tick_mutex_; + std::condition_variable tick_cv_; + std::atomic<bool> running_{false}; + + std::function<void(const std::string &)> on_event_; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/main.cpp b/poc/web_swarm/orchestrator/c++/main.cpp new file mode 100644 index 0000000..4400a0d --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/main.cpp @@ -0,0 +1,205 @@ +#include "control.hpp" +#include "orchestrator.hpp" + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <csignal> +#include <cstdlib> +#include <iostream> +#include <mutex> +#include <string> + +/** + * The swarm orchestrator. + * + * Accepts workers over a WebSocket, negotiates a peer connection, opens a + * control data channel, registers whatever capability the worker reports, and + * keeps the connection under a heartbeat so liveness is a fact rather than an + * assumption. + * + * There is no event loop of our own. libdatachannel runs its own threads and + * the heartbeat runs on another, so main's whole job is to start things, print + * what happens, and wait for a signal. + */ +namespace { + +std::mutex g_wait_mutex; +std::condition_variable g_wait_cv; +std::atomic<bool> g_running{true}; +std::atomic<bool> g_dump_requested{false}; + +/** + * Signal handlers may only touch lock free atomics, so these set a flag and + * poke the condition variable. The notify is outside what the standard blesses + * in a handler, but the predicate is re-checked on a timed wakeup, so a lost + * notify costs at most the poll interval rather than a hang. + */ +extern "C" void handle_stop(int){ + g_running.store(false); + g_wait_cv.notify_all(); +} + +extern "C" void handle_dump(int){ + g_dump_requested.store(true); + g_wait_cv.notify_all(); +} + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +void print_usage(const char *argv0){ + std::cout + << "Usage: " << argv0 << " [options]\n" + << "\n" + << " --port <n> Port for the signalling WebSocket\n" + << " (default 8080). 0 asks the OS for a free one.\n" + << " --stun <url> STUN server, e.g.\n" + << " stun:stun.l.google.com:19302. Not needed on\n" + << " localhost or a LAN.\n" + << " --heartbeat <ms> Ping interval (default 2000).\n" + << " --stale-after <n> Missed pings before a worker is called stale\n" + << " (default 3).\n" + << " --drop-after <n> Missed pings before it is dropped (default 6).\n" + << " --monitor <ms> Print the worker table this often. 0 disables\n" + << " (default 5000). Nothing is printed while no\n" + << " workers are connected.\n" + << " --help This text.\n" + << "\n" + << "SIGUSR1 prints the worker table immediately.\n" + << "\n" + << "The worker page is not served from here -- a WebSocket server is\n" + << "not an HTTP server. Serve it separately, e.g.\n" + << " python3 -m http.server 8000\n"; +} + +} // namespace + +int main(int argc, char **argv){ + using namespace kel::lbm; + + swarm::orchestrator_config ocfg; + swarm::control_config ccfg; + uint64_t monitor_ms = 5000u; + + for(int i = 1; i < argc; ++i){ + const std::string arg{argv[i]}; + + if(arg == "--help" or arg == "-h"){ + print_usage(argv[0]); + return 0; + } + if(arg == "--port" and (i + 1) < argc){ + ocfg.port = static_cast<uint16_t>(std::strtoul(argv[++i], nullptr, 10)); + continue; + } + if(arg == "--stun" and (i + 1) < argc){ + ocfg.stun_url = argv[++i]; + continue; + } + if(arg == "--heartbeat" and (i + 1) < argc){ + ccfg.heartbeat_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--stale-after" and (i + 1) < argc){ + ccfg.limits.stale_after = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--drop-after" and (i + 1) < argc){ + ccfg.limits.drop_after = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--monitor" and (i + 1) < argc){ + monitor_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + + std::cerr << "Unknown or incomplete option: " << arg << "\n\n"; + print_usage(argv[0]); + return 2; + } + + if(ccfg.heartbeat_ms == 0u){ + std::cerr << "--heartbeat must be greater than zero\n"; + return 2; + } + if(ccfg.limits.drop_after < ccfg.limits.stale_after){ + // Otherwise a worker is dropped before it can ever be reported stale, + // which makes the stale state unreachable and the log misleading. + std::cerr << "--drop-after must be at least --stale-after\n"; + return 2; + } + + std::signal(SIGINT, handle_stop); + std::signal(SIGTERM, handle_stop); + std::signal(SIGUSR1, handle_dump); + + swarm::orchestrator orch{ocfg}; + swarm::control ctl{orch, ccfg}; + + ctl.on_event([](const std::string &line){ + std::cout << line << std::endl; + }); + + ctl.attach(); + + uint16_t port = 0u; + try { + port = orch.listen(); + } catch (const std::exception &e){ + std::cerr << "[Error] Could not listen: " << e.what() << "\n"; + return 1; + } + + ctl.start_heartbeat(); + + std::cout << "orchestrator listening on ws://0.0.0.0:" << port << "\n" + << "heartbeat " << ccfg.heartbeat_ms << "ms, stale after " + << ccfg.limits.stale_after << " misses, drop after " + << ccfg.limits.drop_after << "\n" + << "ctrl-c to stop, SIGUSR1 for the worker table" << std::endl; + + uint64_t last_monitor = now_ms(); + + { + std::unique_lock<std::mutex> lock{g_wait_mutex}; + while(g_running.load()){ + g_wait_cv.wait_for(lock, std::chrono::milliseconds{250}, + [](){ return not g_running.load() + or g_dump_requested.load(); }); + + const bool asked = g_dump_requested.exchange(false); + const uint64_t t = now_ms(); + const bool due = (monitor_ms > 0u) and ((t - last_monitor) >= monitor_ms); + + if(not g_running.load()){ + break; + } + if(not asked and not due){ + continue; + } + + last_monitor = t; + + auto snapshot = ctl.workers().snapshot(); + // An idle orchestrator should be quiet; a periodic table of + // nothing is just log noise. An explicit SIGUSR1 always prints, + // because then someone is actually asking. + if(snapshot.empty() and not asked){ + continue; + } + + std::cout << "\n" << snapshot.size() << " worker(s):\n" + << swarm::render_table(snapshot, t) << std::endl; + } + } + + std::cout << "\nstopping" << std::endl; + ctl.stop_heartbeat(); + orch.stop(); + + return 0; +} diff --git a/poc/web_swarm/orchestrator/c++/orchestrator.cpp b/poc/web_swarm/orchestrator/c++/orchestrator.cpp new file mode 100644 index 0000000..68ed8da --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/orchestrator.cpp @@ -0,0 +1,339 @@ +#include "orchestrator.hpp" + +#include <kel/lbm/swarm/signal.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <iostream> +#include <utility> + +namespace kel { +namespace lbm { +namespace swarm { + +namespace { + +/** + * libdatachannel delivers either text or binary. The control and signalling + * protocols are both JSON text, so binary is simply not ours and is dropped. + */ +std::string text_of(const rtc::message_variant &msg){ + if(std::holds_alternative<rtc::string>(msg)){ + return std::get<rtc::string>(msg); + } + return {}; +} + +} // namespace + +orchestrator::orchestrator(orchestrator_config cfg) : cfg_{std::move(cfg)} {} + +orchestrator::~orchestrator(){ + stop(); +} + +void orchestrator::on_message(message_handler handler){ + on_message_ = std::move(handler); +} + +void orchestrator::on_channel_open( + std::function<void(const std::shared_ptr<session> &)> handler){ + on_open_ = std::move(handler); +} + +void orchestrator::on_channel_closed( + std::function<void(const std::shared_ptr<session> &)> handler){ + on_closed_ = std::move(handler); +} + +std::string orchestrator::next_session_id(){ + // Called under the lock. + return "w" + std::to_string(++next_id_); +} + +uint16_t orchestrator::listen(){ + rtc::WebSocketServerConfiguration wscfg; + wscfg.port = cfg_.port; + wscfg.enableTls = false; + + server_ = std::make_unique<rtc::WebSocketServer>(std::move(wscfg)); + server_->onClient([this](std::shared_ptr<rtc::WebSocket> ws){ + accept_client(std::move(ws)); + }); + + return server_->port(); +} + +void orchestrator::accept_client(std::shared_ptr<rtc::WebSocket> ws){ + auto sess = std::make_shared<session>(); + sess->ws = std::move(ws); + + { + std::lock_guard<std::mutex> lock{mutex_}; + sess->id = next_session_id(); + sessions_.emplace(sess->id, sess); + } + + // Callbacks capture weakly. A worker that drops mid-negotiation would + // otherwise keep its own session alive through the capture and never be + // collected. + std::weak_ptr<session> weak = sess; + + sess->ws->onMessage([this, weak](rtc::message_variant msg){ + auto s = weak.lock(); + if(not s){ + return; + } + const std::string text = text_of(msg); + if(text.empty()){ + return; + } + handle_signal(s, text); + }); + + sess->ws->onClosed([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + // The data channel may already be gone; dropping the session is + // idempotent so the two paths cannot double-remove. + drop_session(s->id); + }); + + begin_negotiation(sess); +} + +void orchestrator::begin_negotiation(const std::shared_ptr<session> &sess){ + rtc::Configuration rtccfg; + if(not cfg_.stun_url.empty()){ + rtccfg.iceServers.emplace_back(cfg_.stun_url); + } + + sess->pc = std::make_shared<rtc::PeerConnection>(rtccfg); + + std::weak_ptr<session> weak = sess; + + sess->pc->onLocalDescription([weak](rtc::Description desc){ + auto s = weak.lock(); + if(not s or not s->ws or not s->ws->isOpen()){ + return; + } + + data<sch::SignalDescription> out; + out.get<"type">().set(desc.typeString()); + out.get<"sdp">().set(std::string{desc}); + + auto eo_text = encode_message<sch::SignalDescription>(out); + if(eo_text.is_error()){ + return; + } + s->ws->send(eo_text.get_value()); + }); + + sess->pc->onLocalCandidate([weak](rtc::Candidate cand){ + auto s = weak.lock(); + if(not s or not s->ws or not s->ws->isOpen()){ + return; + } + + data<sch::SignalCandidate> out; + out.get<"type">().set(std::string{sig::candidate}); + out.get<"candidate">().set(cand.candidate()); + out.get<"mid">().set(cand.mid()); + + auto eo_text = encode_message<sch::SignalCandidate>(out); + if(eo_text.is_error()){ + return; + } + s->ws->send(eo_text.get_value()); + }); + + sess->pc->onStateChange([this, weak](rtc::PeerConnection::State state){ + auto s = weak.lock(); + if(not s){ + return; + } + if(state == rtc::PeerConnection::State::Closed or + state == rtc::PeerConnection::State::Failed){ + drop_session(s->id); + } + }); + + /** + * Creating the channel here is what triggers negotiation: with automatic + * negotiation on, the offer is generated because there is now something to + * offer. Doing it from the orchestrator rather than waiting for the worker + * also fixes the ordering -- the channel is in the offer, so the worker + * never has to guess whether it exists yet. + */ + sess->dc = sess->pc->createDataChannel("control"); + + sess->dc->onOpen([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + if(on_open_){ + on_open_(s); + } + }); + + sess->dc->onMessage([this, weak](rtc::message_variant msg){ + auto s = weak.lock(); + if(not s){ + return; + } + const std::string text = text_of(msg); + if(text.empty()){ + return; + } + if(on_message_){ + on_message_(s, text); + } + }); + + sess->dc->onClosed([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + drop_session(s->id); + }); +} + +void orchestrator::handle_signal(const std::shared_ptr<session> &sess, + const std::string &text){ + // Two stage decode, same as the control protocol: learn the type without + // committing to a payload shape. + auto eo_hdr = decode_message<sch::SignalHeader>(text); + if(eo_hdr.is_error()){ + std::cerr << "[" << sess->id << "] malformed signalling message\n"; + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == sig::answer or type == sig::offer){ + auto eo = decode_message<sch::SignalDescription>(text); + if(eo.is_error()){ + std::cerr << "[" << sess->id << "] bad " << type << "\n"; + return; + } + try { + sess->pc->setRemoteDescription( + rtc::Description{eo.get_value().get<"sdp">().stl_string(), type}); + } catch (const std::exception &e){ + std::cerr << "[" << sess->id << "] setRemoteDescription: " + << e.what() << "\n"; + } + return; + } + + if(type == sig::candidate){ + auto eo = decode_message<sch::SignalCandidate>(text); + if(eo.is_error()){ + std::cerr << "[" << sess->id << "] bad candidate\n"; + return; + } + try { + sess->pc->addRemoteCandidate( + rtc::Candidate{eo.get_value().get<"candidate">().stl_string(), + eo.get_value().get<"mid">().stl_string()}); + } catch (const std::exception &e){ + std::cerr << "[" << sess->id << "] addRemoteCandidate: " + << e.what() << "\n"; + } + return; + } + + std::cerr << "[" << sess->id << "] unknown signalling type: " << type << "\n"; +} + +void orchestrator::drop_session(const std::string &id){ + std::shared_ptr<session> sess; + + { + std::lock_guard<std::mutex> lock{mutex_}; + auto iter = sessions_.find(id); + if(iter == sessions_.end()){ + // Already dropped. Several teardown paths converge here -- socket + // closed, channel closed, peer connection failed -- and whichever + // arrives first wins. + return; + } + sess = iter->second; + sessions_.erase(iter); + } + + // Notify outside the lock: the handler may well want to send to other + // sessions, and holding the registry lock across user code is how this + // kind of thing deadlocks later. + if(on_closed_){ + on_closed_(sess); + } +} + +bool orchestrator::send(const std::shared_ptr<session> &sess, + const std::string &text){ + if(not sess or not sess->dc or not sess->dc->isOpen()){ + return false; + } + return sess->dc->send(text); +} + +std::shared_ptr<session> orchestrator::find_session(const std::string &id) const { + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = sessions_.find(id); + if(iter == sessions_.end()){ + return nullptr; + } + + return iter->second; +} + +std::size_t orchestrator::session_count() const { + std::lock_guard<std::mutex> lock{mutex_}; + return sessions_.size(); +} + +void orchestrator::for_each_session( + const std::function<void(const std::shared_ptr<session> &)> &fn) const { + std::lock_guard<std::mutex> lock{mutex_}; + for(const auto &entry : sessions_){ + fn(entry.second); + } +} + +void orchestrator::stop(){ + std::map<std::string, std::shared_ptr<session>> taken; + + { + std::lock_guard<std::mutex> lock{mutex_}; + taken.swap(sessions_); + } + + // Close outside the lock. Closing a peer connection runs callbacks, and + // some of those callbacks want the lock. + for(auto &entry : taken){ + auto &sess = entry.second; + if(sess->dc){ + sess->dc->close(); + } + if(sess->pc){ + sess->pc->close(); + } + if(sess->ws){ + sess->ws->close(); + } + } + + if(server_){ + server_->stop(); + server_.reset(); + } +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/orchestrator.hpp b/poc/web_swarm/orchestrator/c++/orchestrator.hpp new file mode 100644 index 0000000..83ce835 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/orchestrator.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include "session.hpp" + +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <mutex> +#include <string> +#include <string_view> + +namespace kel { +namespace lbm { +namespace swarm { + +struct orchestrator_config { + uint16_t port = 8080u; + + /** + * STUN is not needed on localhost or a LAN -- host candidates are enough, + * and they are gathered without any server. The slot is here so NAT + * traversal is a configuration change later rather than a code change. + */ + std::string stun_url; +}; + +/** + * Accepts workers and holds their connections open. + * + * This is the first code in the repository with concurrency in it, so the rule + * is worth stating rather than inferring: libdatachannel owns the threads and + * delivers every callback on one of them. The only mutable state the + * orchestrator itself owns is the session map, and it is guarded by one mutex. + * Nothing else here is shared. Resist growing that. + * + * A deliberate non-feature: no static file serving. The page has to come from + * somewhere else (python3 -m http.server over the nix output is fine for + * development), because a WebSocket server is not an HTTP server and pretending + * otherwise would mean carrying a second protocol for no benefit. + */ +class orchestrator { +public: + /** + * Called with the session and the decoded text of each control message. + * Handling lives outside this class so the transport can be tested without + * a protocol and the protocol without a transport. + */ + using message_handler = + std::function<void(const std::shared_ptr<session> &, const std::string &)>; + + explicit orchestrator(orchestrator_config cfg); + ~orchestrator(); + + orchestrator(const orchestrator &) = delete; + orchestrator &operator=(const orchestrator &) = delete; + + /// Starts listening. Returns the port actually bound, which differs from + /// the requested one when the config asked for 0. + uint16_t listen(); + + void on_message(message_handler handler); + + /// Invoked once a session's control channel is open and usable. + void on_channel_open(std::function<void(const std::shared_ptr<session> &)> handler); + + /// Invoked when a session goes away, however it goes away. + void on_channel_closed(std::function<void(const std::shared_ptr<session> &)> handler); + + /// Sends text on a session's control channel. False if it is not open. + bool send(const std::shared_ptr<session> &sess, const std::string &text); + + /// Number of sessions currently held. Takes the lock. + std::size_t session_count() const; + + /// Runs `fn` for each session under the lock. Keep it short. + void for_each_session( + const std::function<void(const std::shared_ptr<session> &)> &fn) const; + + /** + * Resolves a session id to something sendable, or null if it has gone. + * + * The registry deals in ids alone so it can stay transport free, which + * leaves this as the one place that maps an id back to a connection. + */ + std::shared_ptr<session> find_session(const std::string &id) const; + + void stop(); + +private: + void accept_client(std::shared_ptr<rtc::WebSocket> ws); + void begin_negotiation(const std::shared_ptr<session> &sess); + void handle_signal(const std::shared_ptr<session> &sess, const std::string &text); + void drop_session(const std::string &id); + + std::string next_session_id(); + + orchestrator_config cfg_; + + std::unique_ptr<rtc::WebSocketServer> server_; + + mutable std::mutex mutex_; + std::map<std::string, std::shared_ptr<session>> sessions_; + uint64_t next_id_ = 0u; + + message_handler on_message_; + std::function<void(const std::shared_ptr<session> &)> on_open_; + std::function<void(const std::shared_ptr<session> &)> on_closed_; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/session.hpp b/poc/web_swarm/orchestrator/c++/session.hpp new file mode 100644 index 0000000..cdc6be3 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/session.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include <rtc/rtc.hpp> + +#include <memory> +#include <mutex> +#include <string> + +namespace kel { +namespace lbm { +namespace swarm { + +/** + * One worker's connection, from the signalling socket through to the control + * channel. + * + * Lifetime is the reason this is a type rather than three locals. libdatachannel + * hands callbacks a reference to nothing in particular, and a PeerConnection + * dropped at the end of the accept handler takes the whole negotiation with it. + * The orchestrator therefore holds a shared_ptr per session and the callbacks + * capture a weak_ptr, so a worker that disconnects mid-negotiation tears down + * cleanly instead of leaving a callback pointing at freed state. + * + * Threading: every member here is touched from libdatachannel's threads, not + * from main. The handles themselves are set once during setup and then only + * read, and the mutable state behind them is libdatachannel's own problem. The + * orchestrator's own mutable state lives in the registry, behind its mutex. + */ +struct session { + /// Assigned by the orchestrator, not the worker. Stable for the connection. + std::string id; + + /// Signalling. Stays open after negotiation so renegotiation is possible. + std::shared_ptr<rtc::WebSocket> ws; + + std::shared_ptr<rtc::PeerConnection> pc; + + /** + * The control channel. Created by the orchestrator rather than the worker + * so that channel-open ordering is deterministic: the offer already + * contains it, so by the time the worker answers it knows the channel is + * coming and does not have to race onDataChannel against its own send. + */ + std::shared_ptr<rtc::DataChannel> dc; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/ui/.nix/derivation.nix b/poc/web_swarm/ui/.nix/derivation.nix deleted file mode 100644 index 8dcabe1..0000000 --- a/poc/web_swarm/ui/.nix/derivation.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ lib -, stdenv -, scons -, clang-tools -, forstio -, python3 -, pname -, version -, adaptive-cpp -, kel -}: - -stdenv.mkDerivation { - pname = pname + "-poc-" + "web_swarm_ui"; - inherit version; - src = ./..; - - nativeBuildInputs = [ - scons - clang-tools - python3 - ]; - - buildInputs = [ - forstio.core - forstio.async - forstio.codec - forstio.codec-unit - forstio.io - forstio.remote - forstio.remote-filesystem - forstio.codec-json - adaptive-cpp - kel.lbm.core - kel.lbm.sycl - ]; - - preferLocalBuild = true; - - outputs = [ "out" "dev" ]; -} diff --git a/poc/web_swarm/ui/html/index.html b/poc/web_swarm/ui/html/index.html deleted file mode 100644 index e69de29..0000000 --- a/poc/web_swarm/ui/html/index.html +++ /dev/null diff --git a/poc/web_swarm/worker/.nix/derivation.nix b/poc/web_swarm/worker/.nix/derivation.nix new file mode 100644 index 0000000..694d951 --- /dev/null +++ b/poc/web_swarm/worker/.nix/derivation.nix @@ -0,0 +1,80 @@ +{ lib +, stdenv +, scons +, clang-tools +, emscripten +, python3 +, nodejs +, pname +, version +, kel +}: + +stdenv.mkDerivation { + pname = "${pname}-web-swarm-worker"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + emscripten + python3 + nodejs + ]; + + # Deliberately empty. em++ is not a nix cc-wrapper, so it never reads + # NIX_CFLAGS_COMPILE and cannot consume buildInputs the way the native + # builds do. Include roots are handed to scons explicitly below, which also + # has the virtue of making the dependency visible rather than ambient. + buildInputs = [ ]; + + sconsFlags = [ + ("kel_include=" + + "${kel.lbm.core.dev}/include:" + + "${kel.lbm.swarm.dev}/include:" + + "${kel.lbm.webgl.dev}/include") + ]; + + # Same class of problem as the AdaptiveCpp HOME hack in + # modules/sycl/.nix/derivation.nix: a toolchain that insists on a writable + # cache inside a read-only build. Seed it from the cache the emscripten + # derivation already precompiled, or every build rebuilds the sysroot. + preBuild = '' + export HOME="$TMPDIR" + export EM_CACHE="$TMPDIR/em_cache" + cp -r ${emscripten}/share/emscripten/cache "$EM_CACHE" + chmod -R u+w "$EM_CACHE" + ''; + + # The real worker needs a browser. Its protocol half does not, so that part + # is built for node and run here; only the GL probe is left uncovered. + doCheck = true; + + checkPhase = '' + runHook preCheck + + node ./bin/kel_lbm_worker_selftest.js + + runHook postCheck + ''; + + postInstall = '' + cat > $out/share/kel-lbm/worker/README <<'EOT' + Serve this directory over HTTP and open index.html. + + python3 -m http.server 8000 + + file:// will not work: wasm is fetched and file:// is a null origin. + + The orchestrator is a separate process on its own port -- a WebSocket + server is not an HTTP server, so it does not serve this page. Start it + with kel_lbm_orchestrator and point the page at it, either in the form + on the page or with ?orchestrator=ws://host:port/ + EOT + ''; + + preferLocalBuild = true; + + outputs = [ "out" ]; +} diff --git a/poc/web_swarm/worker/SConstruct b/poc/web_swarm/worker/SConstruct new file mode 100644 index 0000000..ae0f068 --- /dev/null +++ b/poc/web_swarm/worker/SConstruct @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +# em++ emits a .js loader plus a .wasm sidecar, but SCons only learns about the +# target it was asked for, so the .wasm is invisible to the dependency graph: +# `scons -c` leaves it behind and an incremental build can reuse a stale one. +# +# Program's emitter is an EmitterProxy rather than a dict, so add_emitter() is +# not available on it. Declaring the sidecar explicitly is the supported route: +# SideEffect keeps parallel builds from racing on it, Clean makes -c remove it. +def em_program(env, target, objects): + js = env.Program(target, objects) + wasm = env.File(str(js[0])[:-3] + '.wasm') + env.SideEffect(wasm, js) + env.Clean(js, wasm) + return [js[0], wasm] + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +# em++ is not a nix cc-wrapper, so it never sees NIX_CFLAGS_COMPILE and cannot +# pick up buildInputs the way every other module here does. The derivation +# passes the include roots explicitly instead; see .nix/derivation.nix. +env_vars.Add('kel_include', + help='Colon separated include roots for kel-lbm headers', + default='' +) + +env=Environment(ENV=os.environ, variables=env_vars, + CPPPATH=[p for p in ARGUMENTS.get('kel_include', '').split(':') if p], + CPPDEFINES=[], + CXX = 'em++', + CC = 'emcc', + AR = 'emar', + RANLIB = 'emranlib', + LINK = 'em++', + CXXFLAGS=[ + '-std=c++20', + '-O2', + '-Wall', + '-Wextra', + # kel-lbm's string_literal provides operator""_sl as a string literal + # operator template, which is a GNU extension the library leans on + # deliberately. Without this it warns once per translation unit from a + # core header and drowns out warnings that are actually ours. + '-Wno-gnu-string-literal-operator-template' + ], + LINKFLAGS=[ + # WebGL2 only, and the GLES3 entry points resolved through + # getProcAddress so extension functions work. + '-sFULL_ES3=1', + '-sWASM=1', + '-sMODULARIZE=1', + '-sENVIRONMENT=web', + '-sALLOW_MEMORY_GROWTH=1', + '-sMIN_WEBGL_VERSION=2', + '-sMAX_WEBGL_VERSION=2', + # The page drives the module through these rather than through main. + '-sEXPORTED_FUNCTIONS=_main,_kel_worker_init,_kel_worker_hello,' + '_kel_worker_handle,_kel_worker_pongs,_kel_worker_budget_bytes', + '-sEXPORTED_RUNTIME_METHODS=ccall,cwrap,UTF8ToString' + ], + LIBS=[], + PROGSUFFIX='.js' +); +env.__class__.add_source_files = add_kel_source_files +env.__class__.em_program = em_program + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('c++/SConscript') +SConscript('selftest/SConscript') + +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/poc/web_swarm/worker/c++/SConscript b/poc/web_swarm/worker/c++/SConscript new file mode 100644 index 0000000..a678b7b --- /dev/null +++ b/poc/web_swarm/worker/c++/SConscript @@ -0,0 +1,32 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +worker_env = env.Clone(); + +worker_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +worker_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += worker_env.sources; +env.headers += worker_env.headers; + +objects = [] +worker_env.add_source_files(objects, worker_env.sources, shared=False); +env.worker = worker_env.em_program('#bin/kel_lbm_worker', [objects]); + +env.targets += [env.worker]; + +# The page and its glue live beside the module: index.html loads worker.js for +# the transport and kel_lbm_worker.js for the emscripten loader. +web_dir = Dir('#web').abspath +web_files = sorted(glob.glob(web_dir + "/*")) + +env.Install('$prefix/share/kel-lbm/worker/', env.worker); +env.Install('$prefix/share/kel-lbm/worker/', web_files); diff --git a/poc/web_swarm/worker/c++/protocol.hpp b/poc/web_swarm/worker/c++/protocol.hpp new file mode 100644 index 0000000..16d379f --- /dev/null +++ b/poc/web_swarm/worker/c++/protocol.hpp @@ -0,0 +1,160 @@ +#pragma once + +#include <kel/lbm/webgl/probe.hpp> + +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <cstdint> +#include <string> + +/** + * What the browser worker does, separated from how JavaScript calls it. + * + * worker.cpp is a thin extern "C" shim over this; keeping the behaviour in a + * header means the self test can drive it directly under node, without a + * browser and without going through ccall. The GL probe is the only part that + * genuinely needs a browser, and it fails cleanly when there is no context -- + * which is exactly the state node is in, so everything else stays exercisable. + */ +namespace kel { +namespace lbm { +namespace swarm { + +class worker_protocol { +public: + struct config { + std::string worker_id = "browser"; + std::string user_agent = "unknown"; + }; + + /** + * Runs the capability probe. + * + * Returns false when this device cannot run a solver. That is not a reason + * to stay silent: the worker still registers, so the orchestrator learns + * the device exists and why it is unusable, rather than never hearing from + * it at all. + */ + bool init(config cfg, uint64_t now_ms){ + cfg_ = std::move(cfg); + started_at_ms_ = now_ms; + + webgl::probe_config pcfg; + probe_ok_ = webgl::probe_webgl2(pcfg, capability_); + + return probe_ok_; + } + + /// The hello to send once the control channel opens. + std::string hello(uint64_t now_ms) const { + auto message = make_message<sch::MsgHelloWebGl2>( + msg::hello, cfg_.worker_id, 0u); + + auto &cap = message.get<"capability">(); + cap.get<"worker_id">().set(cfg_.worker_id); + cap.get<"kind">().set(std::string{wkind::webgl2}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(now_ms); + cap.get<"user_agent">().set(cfg_.user_agent); + cap.get<"webgl2">() = capability_; + + auto eo = encode_message<sch::MsgHelloWebGl2>(message); + if(eo.is_error()){ + return {}; + } + return eo.get_value(); + } + + /** + * Handles one control message, returning the reply or an empty string. + * + * The transport never has to know that a ping wants a pong -- only that a + * non-empty return goes back down the channel. + */ + std::string handle(const std::string &text, uint64_t now_ms){ + auto eo_hdr = peek_header(text); + if(eo_hdr.is_error()){ + return {}; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == msg::ping){ + return handle_ping(text, now_ms); + } + if(type == msg::welcome){ + return handle_welcome(now_ms); + } + + // Anything else is not ours to answer. + return {}; + } + + uint64_t pongs() const { return pongs_; } + bool probe_ok() const { return probe_ok_; } + + const data<sch::WebGl2Capability> &capability() const { return capability_; } + +private: + data<sch::WorkerLive> make_live(uint64_t now_ms) const { + data<sch::WorkerLive> live; + live.get<"state">().set(std::string{ + probe_ok_ ? wstate::ready : wstate::error}); + live.get<"uptime_ms">().set( + (now_ms > started_at_ms_) ? (now_ms - started_at_ms_) : 0u); + // Nothing is being simulated yet; the field exists so the monitor does + // not need a protocol change once something is. + live.get<"steps_done">().set(0u); + live.get<"last_error">().set(std::string{""}); + return live; + } + + std::string handle_ping(const std::string &text, uint64_t now_ms){ + auto eo = decode_message<sch::MsgPing>(text); + if(eo.is_error()){ + return {}; + } + + auto pong = make_message<sch::MsgPong>( + msg::pong, cfg_.worker_id, eo.get_value().get<"seq">().get()); + + // Echo the orchestrator's own timestamp so it can measure the round + // trip against its clock. The two machines never have to agree on what + // time it is. + pong.get<"sent_at_ms">().set(eo.get_value().get<"sent_at_ms">().get()); + pong.get<"worker_now_ms">().set(now_ms); + pong.get<"live">() = make_live(now_ms); + + auto eo_out = encode_message<sch::MsgPong>(pong); + if(eo_out.is_error()){ + return {}; + } + + ++pongs_; + return eo_out.get_value(); + } + + std::string handle_welcome(uint64_t now_ms){ + // Registration accepted. Announce readiness, which is a claim only the + // worker can make -- the orchestrator will not assume it. + auto ready = make_message<sch::MsgReady>(msg::ready, cfg_.worker_id, 0u); + ready.get<"live">() = make_live(now_ms); + + auto eo = encode_message<sch::MsgReady>(ready); + if(eo.is_error()){ + return {}; + } + return eo.get_value(); + } + + config cfg_; + data<sch::WebGl2Capability> capability_; + bool probe_ok_ = false; + uint64_t started_at_ms_ = 0u; + uint64_t pongs_ = 0u; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/worker/c++/worker.cpp b/poc/web_swarm/worker/c++/worker.cpp new file mode 100644 index 0000000..db6d2cd --- /dev/null +++ b/poc/web_swarm/worker/c++/worker.cpp @@ -0,0 +1,102 @@ +#include "protocol.hpp" + +#include <emscripten.h> + +#include <string> + +/** + * The browser worker, wasm half: a thin shim over worker_protocol. + * + * The division of labour with worker.js is deliberate and worth stating, + * because the obvious alternative is tempting and wrong. JS owns the + * transport: the WebSocket, the RTCPeerConnection, the data channel and the + * timer. It moves strings and nothing else. Everything that decides what a + * string *means* is on this side. + * + * The reason is single sourcing. The wire format is defined once, as schemas + * in modules/swarm, and both the orchestrator and this worker encode and + * decode through the same codec. A pure JS worker would have been quicker to + * write and would have put a second, hand maintained copy of the format in a + * language where nothing checks it against the first. + * + * Every entry point takes and returns plain C strings. Returned pointers point + * into a static buffer owned here and stay valid until the next call that + * returns a string, which is all JS needs since it copies immediately via + * UTF8ToString. + */ +namespace { + +kel::lbm::swarm::worker_protocol g_worker; + +/// Backing store for whatever was last handed to JS. +std::string g_out; + +const char *hand_over(std::string text){ + g_out = std::move(text); + return g_out.c_str(); +} + +/** + * Monotonic milliseconds since the page started. Deliberately performance.now + * rather than wall clock: the live block reports an uptime, and round trips + * are measured by echoing the orchestrator's own stamp, so nothing here needs + * to agree with another machine's idea of the time. + */ +EM_JS(double, js_now_ms, (), { + return performance.now(); +}); + +uint64_t now_ms(){ + return static_cast<uint64_t>(js_now_ms()); +} + +} // namespace + +extern "C" { + +/// Runs the capability probe. Call once, before connecting. 1 if this device +/// can actually run a solver, 0 otherwise. +EMSCRIPTEN_KEEPALIVE +int kel_worker_init(const char *worker_id, const char *user_agent){ + kel::lbm::swarm::worker_protocol::config cfg; + if(worker_id != nullptr and *worker_id != '\0'){ + cfg.worker_id = worker_id; + } + if(user_agent != nullptr){ + cfg.user_agent = user_agent; + } + + return g_worker.init(std::move(cfg), now_ms()) ? 1 : 0; +} + +EMSCRIPTEN_KEEPALIVE +const char *kel_worker_hello(){ + return hand_over(g_worker.hello(now_ms())); +} + +EMSCRIPTEN_KEEPALIVE +const char *kel_worker_handle(const char *text){ + if(text == nullptr){ + return hand_over(std::string{}); + } + return hand_over(g_worker.handle(std::string{text}, now_ms())); +} + +EMSCRIPTEN_KEEPALIVE +double kel_worker_pongs(){ + return static_cast<double>(g_worker.pongs()); +} + +EMSCRIPTEN_KEEPALIVE +double kel_worker_budget_bytes(){ + return static_cast<double>( + g_worker.capability().get<"budget">().get<"probed_bytes_safe">().get()); +} + +} // extern "C" + +int main(){ + // Nothing to do. The page drives everything through the exports above; + // main exists because emscripten wants an entry point. + return 0; +} diff --git a/poc/web_swarm/worker/selftest/SConscript b/poc/web_swarm/worker/selftest/SConscript new file mode 100644 index 0000000..f60d79a --- /dev/null +++ b/poc/web_swarm/worker/selftest/SConscript @@ -0,0 +1,35 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Built for node rather than the web, so a check phase can run it. Same +# protocol code as the real worker; only the entry point and the environment +# differ. +test_env = env.Clone(); +test_env['LINKFLAGS'] = [ + '-sWASM=1', + '-sENVIRONMENT=node', + '-sALLOW_MEMORY_GROWTH=1', + # The probe is compiled in and must link, but with no context it returns + # cleanly rather than running any of it. + '-sMIN_WEBGL_VERSION=2', + '-sMAX_WEBGL_VERSION=2', + '-sFULL_ES3=1', +] + +test_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); + +env.sources += test_env.sources; + +objects = [] +test_env.add_source_files(objects, test_env.sources, shared=False); +env.selftest = test_env.em_program('#bin/kel_lbm_worker_selftest', [objects]); + +env.targets += [env.selftest]; diff --git a/poc/web_swarm/worker/selftest/selftest.cpp b/poc/web_swarm/worker/selftest/selftest.cpp new file mode 100644 index 0000000..ee13bbf --- /dev/null +++ b/poc/web_swarm/worker/selftest/selftest.cpp @@ -0,0 +1,146 @@ +#include "../c++/protocol.hpp" + +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <cstdio> +#include <string> + +/** + * Node-hosted self test for the browser worker's protocol half. + * + * A check phase has no browser, so the real worker cannot be tested there. + * But only the GL probe actually needs one -- the part that decides what to + * say, and says it in the format the orchestrator parses, is ordinary code. + * Built for node and run in the derivation's check phase, this covers that + * part and leaves only the probe genuinely untested. + * + * Running with no WebGL context is not a limitation here, it is the point: it + * is the same path a browser takes when the device cannot give one, and the + * worker has to register anyway so the orchestrator learns why. + */ +namespace { + +using namespace kel::lbm; + +int failures = 0; + +void check(bool ok, const char *what){ + if(ok){ + std::printf(" ok %s\n", what); + return; + } + std::printf(" FAIL %s\n", what); + ++failures; +} + +} // namespace + +int main(){ + std::printf("worker protocol self test\n"); + + swarm::worker_protocol worker; + + swarm::worker_protocol::config cfg; + cfg.worker_id = "selftest"; + cfg.user_agent = "node"; + + // No WebGL under node, so the probe must fail -- cleanly, without taking + // the rest of the worker down with it. + const bool usable = worker.init(cfg, 1000u); + check(not usable, "probe reports the device unusable without a context"); + check(not worker.probe_ok(), "and remembers that it did"); + + // Registration still happens. An orchestrator that knows why a device is + // unusable is better off than one that never heard from it. + const std::string hello = worker.hello(2000u); + check(not hello.empty(), "an unusable device still produces a hello"); + + { + auto eo = swarm::decode_message<sch::MsgHelloWebGl2>(hello); + check(not eo.is_error(), "the hello decodes as a webgl2 hello"); + if(not eo.is_error()){ + const auto &cap = eo.get_value().get<"capability">(); + check(cap.get<"kind">().stl_view() == wkind::webgl2, + "it registers as a webgl2 worker"); + check(cap.get<"worker_id">().stl_view() == "selftest", + "it carries the configured id"); + check(cap.get<"schema_version">().stl_view() == schema_version, + "it states the schema version the orchestrator checks"); + check(not cap.get<"webgl2">().get<"features">() + .get<"color_buffer_float">().get(), + "and honestly reports no float render target"); + } + } + + // The routing decode the orchestrator does first has to work on it too. + { + auto eo = swarm::decode_message<sch::MsgHello>(hello); + check(not eo.is_error(), "the orchestrator's routing decode accepts it"); + } + + // A ping must come back as a pong echoing the sequence and the timestamp. + { + auto ping = swarm::make_message<sch::MsgPing>(msg::ping, "orch", 42u); + ping.get<"sent_at_ms">().set(123456u); + + auto eo_ping = swarm::encode_message<sch::MsgPing>(ping); + check(not eo_ping.is_error(), "a ping encodes"); + + const std::string reply = worker.handle(eo_ping.get_value(), 3000u); + check(not reply.empty(), "a ping is answered"); + + auto eo_pong = swarm::decode_message<sch::MsgPong>(reply); + check(not eo_pong.is_error(), "the answer is a well formed pong"); + if(not eo_pong.is_error()){ + const auto &pong = eo_pong.get_value(); + check(pong.get<"type">().stl_view() == msg::pong, "typed as a pong"); + check(pong.get<"seq">().get() == 42u, + "echoing the sequence, so a late pong can be told apart"); + check(pong.get<"sent_at_ms">().get() == 123456u, + "echoing the timestamp, so rtt needs no clock agreement"); + check(pong.get<"worker_now_ms">().get() == 3000u, + "and carrying the worker's own clock"); + check(pong.get<"live">().get<"state">().stl_view() == wstate::error, + "reporting error state, since the probe failed"); + } + + check(worker.pongs() == 1u, "the pong is counted"); + } + + // A welcome must produce a ready. Readiness is a claim only the worker + // makes; the orchestrator never assumes it. + { + auto welcome = swarm::make_message<sch::MsgWelcome>( + msg::welcome, "w1", 0u); + welcome.get<"heartbeat_ms">().set(2000u); + welcome.get<"schema_version">().set(std::string{schema_version}); + + auto eo_wel = swarm::encode_message<sch::MsgWelcome>(welcome); + check(not eo_wel.is_error(), "a welcome encodes"); + + const std::string reply = worker.handle(eo_wel.get_value(), 4000u); + check(not reply.empty(), "a welcome is answered"); + + auto eo_ready = swarm::decode_message<sch::MsgReady>(reply); + check(not eo_ready.is_error(), "the answer is a well formed ready"); + if(not eo_ready.is_error()){ + check(eo_ready.get_value().get<"type">().stl_view() == msg::ready, + "typed as ready"); + } + } + + // Things the worker has no business answering must produce nothing rather + // than an empty message on the wire. + { + check(worker.handle("", 5000u).empty(), "garbage is not answered"); + check(worker.handle("{", 5000u).empty(), "malformed json is not answered"); + check(worker.handle(R"({"type":"pong","worker_id":"x","seq":1})", 5000u) + .empty(), + "a message meant for the orchestrator is not answered"); + check(worker.pongs() == 1u, "and none of that counted as a pong"); + } + + std::printf("%s\n", failures == 0 ? "PASS" : "FAIL"); + return failures == 0 ? 0 : 1; +} diff --git a/poc/web_swarm/worker/web/index.html b/poc/web_swarm/worker/web/index.html new file mode 100644 index 0000000..dddf4a1 --- /dev/null +++ b/poc/web_swarm/worker/web/index.html @@ -0,0 +1,87 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>kel-lbm swarm worker</title> +<style> + body { + margin: 0; + padding: 1.5rem; + background: #14161a; + color: #d8dee9; + font-family: system-ui, sans-serif; + line-height: 1.5; + } + h1 { font-size: 1.1rem; margin: 0 0 1rem; font-weight: 600; } + .row { display: flex; gap: 0.5rem; margin-bottom: 0.5rem; flex-wrap: wrap; } + label { min-width: 7rem; align-self: center; font-size: 0.85rem; color: #8c95a3; } + input { + flex: 1; min-width: 14rem; padding: 0.4rem 0.6rem; + background: #1d2027; color: #d8dee9; + border: 1px solid #333842; border-radius: 4px; + font-family: ui-monospace, monospace; font-size: 0.85rem; + } + button { + padding: 0.4rem 1rem; border-radius: 4px; cursor: pointer; + background: #2d4f74; color: #e8eef5; + border: 1px solid #3b6694; font-size: 0.85rem; + } + button:disabled { opacity: 0.4; cursor: default; } + #status { + margin: 1rem 0; padding: 0.75rem 1rem; + background: #1d2027; border-left: 3px solid #3b6694; border-radius: 3px; + } + #state { font-size: 1.05rem; font-weight: 600; } + #detail { color: #8c95a3; font-size: 0.85rem; font-family: ui-monospace, monospace; } + #log { + height: 16rem; overflow-y: auto; white-space: pre-wrap; + background: #0f1115; border: 1px solid #262b33; border-radius: 4px; + padding: 0.75rem; font-family: ui-monospace, monospace; font-size: 0.78rem; + color: #9aa5b1; + } + /* + * The probe needs a real canvas to get a WebGL2 context from, but nothing is + * ever drawn for display. Kept at one pixel and out of the way rather than + * display:none, which some browsers treat as a reason to skip the context. + */ + #kel-probe { position: absolute; width: 1px; height: 1px; opacity: 0; pointer-events: none; } + .note { color: #6b7684; font-size: 0.78rem; margin-top: 1rem; } +</style> +</head> +<body> + +<h1>kel-lbm swarm worker</h1> + +<div class="row"> + <label for="url">orchestrator</label> + <input id="url" type="text" spellcheck="false"> +</div> +<div class="row"> + <label for="worker-id">worker id</label> + <input id="worker-id" type="text" spellcheck="false"> +</div> +<div class="row"> + <button id="connect" disabled>connect</button> + <button id="disconnect" disabled>disconnect</button> +</div> + +<div id="status"> + <div id="state">loading</div> + <div id="detail"></div> +</div> + +<div id="log"></div> + +<p class="note"> + Open this page from an HTTP server, not a file:// URL — wasm is fetched, + and file:// is a null origin. The orchestrator is a separate process on its own + port; append <code>?orchestrator=ws://host:port/</code> to override. +</p> + +<canvas id="kel-probe" width="1" height="1"></canvas> + +<script src="worker.js"></script> +<script src="kel_lbm_worker.js"></script> +</body> +</html> diff --git a/poc/web_swarm/worker/web/worker.js b/poc/web_swarm/worker/web/worker.js new file mode 100644 index 0000000..04d4e10 --- /dev/null +++ b/poc/web_swarm/worker/web/worker.js @@ -0,0 +1,286 @@ +// The browser worker, transport half. +// +// This file owns the WebSocket, the RTCPeerConnection, the data channel and +// nothing else. Every decision about what a message means lives in the wasm +// module, so the wire format has exactly one definition -- the schemas in +// modules/swarm -- rather than one in C++ and a hand maintained copy here. +// +// The rule to keep: if you find yourself about to read a field out of a parsed +// message in this file, it belongs on the other side of the boundary. + +'use strict'; + +const ui = { + state: null, + detail: null, + log: null, +}; + +function setState(text, detail) { + if (ui.state) ui.state.textContent = text; + if (ui.detail && detail !== undefined) ui.detail.textContent = detail; +} + +function log(line) { + console.log(line); + if (!ui.log) return; + const stamp = new Date().toLocaleTimeString(); + ui.log.textContent += `[${stamp}] ${line}\n`; + ui.log.scrollTop = ui.log.scrollHeight; +} + +// Normalises an ICE candidate into the shape modules/swarm declares. The +// browser hands out {candidate, sdpMid, sdpMLineIndex}; the wire carries +// {type, candidate, mid}. Doing the mapping here keeps every message on the +// wire uniform -- all of them are objects with a "type". +function candidateMessage(candidate) { + return JSON.stringify({ + type: 'candidate', + candidate: candidate.candidate, + mid: candidate.sdpMid === null ? '' : candidate.sdpMid, + }); +} + +function descriptionMessage(description) { + return JSON.stringify({ + type: description.type, + sdp: description.sdp, + }); +} + +class Worker { + constructor(mod, url, workerId) { + this.mod = mod; + this.url = url; + this.workerId = workerId; + this.ws = null; + this.pc = null; + this.control = null; + + // Bound once so the wasm exports are called by name rather than through + // ccall at every message, which would re-resolve them each time. + this.hello = mod.cwrap('kel_worker_hello', 'string', []); + this.handle = mod.cwrap('kel_worker_handle', 'string', ['string']); + this.pongs = mod.cwrap('kel_worker_pongs', 'number', []); + } + + connect() { + setState('connecting', this.url); + log(`connecting to ${this.url}`); + + this.ws = new WebSocket(this.url); + + this.ws.onopen = () => { + setState('signalling'); + log('signalling socket open'); + }; + + this.ws.onerror = () => { + setState('error', 'signalling socket failed'); + log('signalling socket error'); + }; + + this.ws.onclose = () => { + setState('disconnected'); + log('signalling socket closed'); + }; + + this.ws.onmessage = (event) => this.onSignal(event.data); + + this.startPeerConnection(); + } + + startPeerConnection() { + // No ICE servers: host candidates are enough on localhost or a LAN, and + // a STUN server is only needed to get through NAT. + this.pc = new RTCPeerConnection({ iceServers: [] }); + + this.pc.onicecandidate = (event) => { + if (!event.candidate) return; + this.send(candidateMessage(event.candidate)); + }; + + this.pc.onconnectionstatechange = () => { + log(`peer connection ${this.pc.connectionState}`); + if (this.pc.connectionState === 'failed') { + setState('error', 'peer connection failed'); + } + }; + + // The orchestrator creates the channel, so it arrives rather than being + // made here. That fixes the ordering: it is already in the offer, so + // there is no race between it opening and the first send. + this.pc.ondatachannel = (event) => { + if (event.channel.label !== 'control') return; + this.attachControl(event.channel); + }; + } + + attachControl(channel) { + this.control = channel; + + channel.onopen = () => { + setState('registering'); + log('control channel open'); + + const hello = this.hello(); + if (!hello) { + setState('error', 'could not build hello'); + return; + } + log(`-> hello (${hello.length} bytes)`); + channel.send(hello); + }; + + channel.onmessage = (event) => { + if (typeof event.data !== 'string') return; + + // The one place the transport looks inside a message, and only to say + // what happened in the log. The reply itself is decided by wasm. + const reply = this.handle(event.data); + + let kind = '?'; + try { + kind = JSON.parse(event.data).type; + } catch (err) { + kind = 'unparseable'; + } + + if (kind === 'welcome') { + setState('registered', `pongs 0`); + log('<- welcome, registered'); + } else if (kind === 'ping') { + setState('registered', `pongs ${this.pongs()}`); + } else { + log(`<- ${kind}`); + } + + if (reply && channel.readyState === 'open') { + channel.send(reply); + } + }; + + channel.onclose = () => { + setState('disconnected', 'control channel closed'); + log('control channel closed'); + }; + } + + onSignal(text) { + let message; + try { + message = JSON.parse(text); + } catch (err) { + log('malformed signalling message'); + return; + } + + if (message.type === 'offer' || message.type === 'answer') { + this.pc + .setRemoteDescription({ type: message.type, sdp: message.sdp }) + .then(() => { + if (message.type !== 'offer') return null; + return this.pc.createAnswer(); + }) + .then((answer) => { + if (!answer) return null; + return this.pc.setLocalDescription(answer).then(() => { + this.send(descriptionMessage(this.pc.localDescription)); + }); + }) + .catch((err) => log(`signalling failed: ${err}`)); + return; + } + + if (message.type === 'candidate') { + this.pc + .addIceCandidate({ + candidate: message.candidate, + sdpMid: message.mid === '' ? null : message.mid, + }) + .catch((err) => log(`bad candidate: ${err}`)); + return; + } + + log(`unknown signalling type: ${message.type}`); + } + + send(text) { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(text); + } + } + + disconnect() { + if (this.control) this.control.close(); + if (this.pc) this.pc.close(); + if (this.ws) this.ws.close(); + setState('disconnected'); + } +} + +function defaultUrl() { + const params = new URLSearchParams(window.location.search); + const fromQuery = params.get('orchestrator'); + if (fromQuery) return fromQuery; + // The page is served by something else -- a WebSocket server is not an HTTP + // server -- so the orchestrator is not on this port. 8080 is its default. + return `ws://${window.location.hostname || '127.0.0.1'}:8080/`; +} + +window.addEventListener('DOMContentLoaded', () => { + ui.state = document.getElementById('state'); + ui.detail = document.getElementById('detail'); + ui.log = document.getElementById('log'); + + const urlInput = document.getElementById('url'); + const idInput = document.getElementById('worker-id'); + const connectButton = document.getElementById('connect'); + const disconnectButton = document.getElementById('disconnect'); + + urlInput.value = defaultUrl(); + idInput.value = `browser-${Math.floor(Math.random() * 10000)}`; + + setState('loading', 'instantiating wasm'); + + Module().then((mod) => { + const init = mod.cwrap('kel_worker_init', 'number', ['string', 'string']); + const budget = mod.cwrap('kel_worker_budget_bytes', 'number', []); + + log('wasm ready, probing device'); + + const usable = init(idInput.value, navigator.userAgent); + const mib = Math.round(budget() / (1024 * 1024)); + + if (usable) { + setState('probed', `${mib} MiB usable`); + log(`probe: usable, ${mib} MiB safe budget`); + } else { + // Still worth connecting. An orchestrator that knows why a device is + // unusable is better off than one that never heard from it. + setState('probed', 'device cannot run a solver'); + log('probe: device unusable (no float render target, or no context)'); + } + + let worker = null; + + connectButton.disabled = false; + connectButton.onclick = () => { + if (worker) worker.disconnect(); + worker = new Worker(mod, urlInput.value, idInput.value); + worker.connect(); + connectButton.disabled = true; + disconnectButton.disabled = false; + }; + + disconnectButton.onclick = () => { + if (worker) worker.disconnect(); + worker = null; + connectButton.disabled = false; + disconnectButton.disabled = true; + }; + }).catch((err) => { + setState('error', 'wasm failed to load'); + log(`wasm load failed: ${err}`); + }); +}); diff --git a/poc/web_swarm/worker_native/c++/SConscript b/poc/web_swarm/worker_native/c++/SConscript new file mode 100644 index 0000000..873e075 --- /dev/null +++ b/poc/web_swarm/worker_native/c++/SConscript @@ -0,0 +1,26 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +worker_env = env.Clone(); + +worker_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +worker_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += worker_env.sources; +env.headers += worker_env.headers; + +objects = [] +worker_env.add_source_files(objects, worker_env.sources, shared=False); +env.worker_native = worker_env.Program('#bin/kel_lbm_worker_native', [objects]); + +env.targets += [env.worker_native]; + +env.Install('$prefix/bin/', env.worker_native); diff --git a/poc/web_swarm/worker_native/c++/capability.hpp b/poc/web_swarm/worker_native/c++/capability.hpp new file mode 100644 index 0000000..d892a8f --- /dev/null +++ b/poc/web_swarm/worker_native/c++/capability.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include <kel/lbm/swarm/capability.hpp> +#include <kel/lbm/abstract/data.hpp> + +#include <string> +#include <thread> + +#if defined(__linux__) +#include <unistd.h> +#endif + +namespace kel { +namespace lbm { +namespace swarm { + +/** + * What a native worker can honestly say about itself. + * + * Deliberately short. A browser worker has to measure its texture budget + * because WebGL will not tell it, but a process can simply ask the OS, and + * there is no point inventing numbers beyond that until there is a scheduler + * that would use them. + */ +inline data<sch::NativeCapability> probe_native_capability(){ + data<sch::NativeCapability> cap; + + unsigned threads = std::thread::hardware_concurrency(); + if(threads == 0u){ + // hardware_concurrency is allowed to give up; one thread is the + // honest floor rather than zero, which would read as "no capacity". + threads = 1u; + } + cap.get<"threads">().set(threads); + + uint64_t memory_bytes = 0u; +#if defined(__linux__) + const long pages = ::sysconf(_SC_PHYS_PAGES); + const long page_size = ::sysconf(_SC_PAGE_SIZE); + if(pages > 0 and page_size > 0){ + memory_bytes = static_cast<uint64_t>(pages) * static_cast<uint64_t>(page_size); + } +#endif + cap.get<"memory_bytes">().set(memory_bytes); + +#if defined(__x86_64__) + cap.get<"cpu">().set(std::string{"x86_64"}); +#elif defined(__aarch64__) + cap.get<"cpu">().set(std::string{"aarch64"}); +#else + cap.get<"cpu">().set(std::string{"unknown"}); +#endif + + // Nothing here links SYCL yet. Reported so the orchestrator can start + // distinguishing compute capable workers without a protocol change. + cap.get<"has_sycl">().set(false); + + return cap; +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/worker_native/c++/main.cpp b/poc/web_swarm/worker_native/c++/main.cpp new file mode 100644 index 0000000..6f4fbc8 --- /dev/null +++ b/poc/web_swarm/worker_native/c++/main.cpp @@ -0,0 +1,403 @@ +#include "capability.hpp" + +#include <rtc/rtc.hpp> + +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/signal.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <cstdlib> +#include <iostream> +#include <memory> +#include <mutex> +#include <string> + +/** + * A native worker. + * + * Two jobs, and it is worth being explicit that the second is not a lesser one: + * + * 1. It is a worker kind. A process can join the swarm and be handed work, + * and it registers as kind "native" with the capability a process can + * actually report -- threads and memory -- rather than impersonating a + * browser and claiming a texture budget it has no way to measure. + * + * 2. It is the only end to end coverage the orchestrator can have in CI. The + * browser worker needs a browser, which a nix check phase does not have, + * so this is what keeps the signalling, the peer connection and the + * control channel under test. + * + * The transport is the mirror image of the orchestrator's: connect the + * signalling socket, answer the offer, wait for the channel the orchestrator + * created, then speak the control protocol on it. + */ +namespace { + +using namespace kel::lbm; + +struct worker_config { + std::string url = "ws://127.0.0.1:8080/"; + std::string worker_id = "native"; + uint64_t timeout_ms = 15000u; + + /** + * Exit once the hello has been answered. This is what the check script + * runs; left off, the worker stays connected like a real one. + */ + bool once = false; + + /** + * Exit after answering this many pings. Zero means never. Takes + * precedence over --once, so a check can cover registration and liveness + * in a single run rather than two. + */ + uint64_t pongs_target = 0u; +}; + +std::atomic<uint64_t> g_pongs_sent{0u}; + +std::mutex g_mutex; +std::condition_variable g_cv; +std::atomic<bool> g_done{false}; +std::atomic<int> g_exit_code{1}; + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +void finish(int code){ + g_exit_code.store(code); + g_done.store(true); + g_cv.notify_all(); +} + +std::string text_of(const rtc::message_variant &msg){ + if(std::holds_alternative<rtc::string>(msg)){ + return std::get<rtc::string>(msg); + } + return {}; +} + +data<sch::WorkerLive> make_live(uint64_t started_at){ + data<sch::WorkerLive> live; + live.get<"state">().set(std::string{wstate::ready}); + live.get<"uptime_ms">().set(now_ms() - started_at); + live.get<"steps_done">().set(0u); + live.get<"last_error">().set(std::string{""}); + return live; +} + +void print_usage(const char *argv0){ + std::cout + << "Usage: " << argv0 << " [options]\n" + << "\n" + << " --url <ws url> Orchestrator signalling socket\n" + << " (default ws://127.0.0.1:8080/).\n" + << " --id <name> Worker id to report (default 'native').\n" + << " --timeout <ms> Give up if registration has not completed\n" + << " (default 15000).\n" + << " --once Exit 0 once the hello has been answered.\n" + << " Used by the check; without it the worker\n" + << " stays connected.\n" + << " --pongs <n> Exit 0 after answering n pings. Takes\n" + << " precedence over --once, so one run can cover\n" + << " registration and liveness together.\n" + << " --help This text.\n"; +} + +} // namespace + +int main(int argc, char **argv){ + worker_config cfg; + + for(int i = 1; i < argc; ++i){ + const std::string arg{argv[i]}; + + if(arg == "--help" or arg == "-h"){ + print_usage(argv[0]); + return 0; + } + if(arg == "--url" and (i + 1) < argc){ + cfg.url = argv[++i]; + continue; + } + if(arg == "--id" and (i + 1) < argc){ + cfg.worker_id = argv[++i]; + continue; + } + if(arg == "--timeout" and (i + 1) < argc){ + cfg.timeout_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--once"){ + cfg.once = true; + continue; + } + if(arg == "--pongs" and (i + 1) < argc){ + cfg.pongs_target = std::strtoull(argv[++i], nullptr, 10); + continue; + } + + std::cerr << "Unknown or incomplete option: " << arg << "\n\n"; + print_usage(argv[0]); + return 2; + } + + const uint64_t started_at = now_ms(); + + auto ws = std::make_shared<rtc::WebSocket>(); + auto pc = std::make_shared<rtc::PeerConnection>(); + + /** + * The orchestrator creates the control channel, so the worker waits for it + * rather than making one. Held here so the callbacks below outlive the + * negotiation. + */ + std::shared_ptr<rtc::DataChannel> control; + std::mutex control_mutex; + + pc->onLocalDescription([ws](rtc::Description desc){ + if(not ws->isOpen()){ + return; + } + data<sch::SignalDescription> out; + out.get<"type">().set(desc.typeString()); + out.get<"sdp">().set(std::string{desc}); + + auto eo = swarm::encode_message<sch::SignalDescription>(out); + if(eo.is_error()){ + return; + } + ws->send(eo.get_value()); + }); + + pc->onLocalCandidate([ws](rtc::Candidate cand){ + if(not ws->isOpen()){ + return; + } + data<sch::SignalCandidate> out; + out.get<"type">().set(std::string{sig::candidate}); + out.get<"candidate">().set(cand.candidate()); + out.get<"mid">().set(cand.mid()); + + auto eo = swarm::encode_message<sch::SignalCandidate>(out); + if(eo.is_error()){ + return; + } + ws->send(eo.get_value()); + }); + + pc->onStateChange([](rtc::PeerConnection::State state){ + if(state == rtc::PeerConnection::State::Failed){ + std::cerr << "peer connection failed\n"; + finish(1); + } + }); + + pc->onDataChannel([&, started_at](std::shared_ptr<rtc::DataChannel> dc){ + if(dc->label() != "control"){ + // Nothing else is expected; ignoring rather than failing keeps the + // worker forward compatible with an orchestrator that adds one. + return; + } + + { + std::lock_guard<std::mutex> lock{control_mutex}; + control = dc; + } + + dc->onOpen([dc, &cfg, started_at](){ + std::cout << "control channel open" << std::endl; + + auto hello = swarm::make_message<sch::MsgHelloNative>( + msg::hello, cfg.worker_id, 0u); + + auto &cap = hello.get<"capability">(); + cap.get<"worker_id">().set(cfg.worker_id); + cap.get<"kind">().set(std::string{wkind::native}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(now_ms()); + cap.get<"user_agent">().set(std::string{"kel-lbm-worker-native"}); + cap.get<"native">() = swarm::probe_native_capability(); + + auto eo = swarm::encode_message<sch::MsgHelloNative>(hello); + if(eo.is_error()){ + std::cerr << "could not encode hello\n"; + finish(1); + return; + } + + std::cout << "-> hello (" << eo.get_value().size() << " bytes)" + << std::endl; + dc->send(eo.get_value()); + }); + + dc->onMessage([dc, &cfg, started_at](rtc::message_variant raw){ + const std::string text = text_of(raw); + if(text.empty()){ + return; + } + + auto eo_hdr = swarm::peek_header(text); + if(eo_hdr.is_error()){ + std::cerr << "malformed control message\n"; + finish(1); + return; + } + + const std::string type = + eo_hdr.get_value().get<"type">().stl_string(); + std::cout << "<- " << type << std::endl; + + /** + * Step 2's orchestrator echoes rather than replying, so a hello + * coming back is what confirms the round trip. Once the registry + * lands this becomes a welcome, and both are accepted so the two + * can be developed in either order. + */ + if(type == msg::hello or type == msg::welcome){ + // Registration is done. If the run is also meant to cover + // liveness, stay for the pings instead of leaving now. + if(cfg.once and cfg.pongs_target == 0u){ + finish(0); + } + return; + } + + if(type == msg::ping){ + auto eo_ping = swarm::decode_message<sch::MsgPing>(text); + if(eo_ping.is_error()){ + std::cerr << "bad ping\n"; + return; + } + + auto pong = swarm::make_message<sch::MsgPong>( + msg::pong, cfg.worker_id, + eo_ping.get_value().get<"seq">().get()); + pong.get<"sent_at_ms">().set( + eo_ping.get_value().get<"sent_at_ms">().get()); + pong.get<"worker_now_ms">().set(now_ms()); + pong.get<"live">() = make_live(started_at); + + auto eo_out = swarm::encode_message<sch::MsgPong>(pong); + if(eo_out.is_error()){ + return; + } + + // Reply on the channel the message arrived on rather than + // looking it up: same channel, no lock, no lifetime question. + if(dc->isOpen()){ + dc->send(eo_out.get_value()); + const uint64_t sent = g_pongs_sent.fetch_add(1u) + 1u; + if(cfg.pongs_target > 0u and sent >= cfg.pongs_target){ + std::cout << "answered " << sent << " ping(s)" + << std::endl; + finish(0); + } + } + return; + } + }); + + dc->onClosed([](){ + std::cout << "control channel closed" << std::endl; + finish(g_exit_code.load()); + }); + }); + + ws->onMessage([pc](rtc::message_variant raw){ + const std::string text = text_of(raw); + if(text.empty()){ + return; + } + + auto eo_hdr = swarm::decode_message<sch::SignalHeader>(text); + if(eo_hdr.is_error()){ + std::cerr << "malformed signalling message\n"; + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == sig::offer or type == sig::answer){ + auto eo = swarm::decode_message<sch::SignalDescription>(text); + if(eo.is_error()){ + std::cerr << "bad " << type << "\n"; + return; + } + try { + // Auto negotiation turns setting a remote offer into an answer. + pc->setRemoteDescription( + rtc::Description{eo.get_value().get<"sdp">().stl_string(), type}); + } catch (const std::exception &e){ + std::cerr << "setRemoteDescription: " << e.what() << "\n"; + } + return; + } + + if(type == sig::candidate){ + auto eo = swarm::decode_message<sch::SignalCandidate>(text); + if(eo.is_error()){ + std::cerr << "bad candidate\n"; + return; + } + try { + pc->addRemoteCandidate( + rtc::Candidate{eo.get_value().get<"candidate">().stl_string(), + eo.get_value().get<"mid">().stl_string()}); + } catch (const std::exception &e){ + std::cerr << "addRemoteCandidate: " << e.what() << "\n"; + } + return; + } + }); + + ws->onClosed([](){ + // Closing before the exchange completed is a failure; after it, the + // exit code is already set and this is just teardown. + finish(g_exit_code.load()); + }); + + ws->onError([](std::string err){ + std::cerr << "signalling error: " << err << "\n"; + finish(1); + }); + + std::cout << "connecting to " << cfg.url << std::endl; + try { + ws->open(cfg.url); + } catch (const std::exception &e){ + std::cerr << "[Error] Could not open " << cfg.url << ": " << e.what() << "\n"; + return 1; + } + + { + std::unique_lock<std::mutex> lock{g_mutex}; + const bool finished = g_cv.wait_for( + lock, std::chrono::milliseconds{cfg.timeout_ms}, + [](){ return g_done.load(); }); + + if(not finished){ + std::cerr << "[Error] Timed out after " << cfg.timeout_ms + << "ms without completing registration\n"; + g_exit_code.store(1); + } + } + + // Tear down before the callbacks' captures go out of scope. + { + std::lock_guard<std::mutex> lock{control_mutex}; + if(control){ + control->close(); + } + } + pc->close(); + ws->close(); + + return g_exit_code.load(); +} |
