diff options
Diffstat (limited to 'poc/web_swarm')
27 files changed, 2802 insertions, 114 deletions
diff --git a/poc/web_swarm/.nix/derivation.nix b/poc/web_swarm/.nix/derivation.nix new file mode 100644 index 0000000..31f1e2b --- /dev/null +++ b/poc/web_swarm/.nix/derivation.nix @@ -0,0 +1,52 @@ +{ lib +, stdenv +, scons +, clang-tools +, libdatachannel +, pname +, version +, kel +}: + +stdenv.mkDerivation { + pname = "${pname}-web-swarm"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + # Ordinary native derivations, so unlike the webgl module these get their + # include paths from nix the usual way. + # + # libdatachannel ships CMake files but no pkg-config, hence the plain + # LIBS=['datachannel'] in the SConstruct rather than a pkg-config lookup. + # Note there is no forstio here: libdatachannel owns the threads and the + # event handling, and forstio has no WebSocket or WebRTC to offer. + buildInputs = [ + libdatachannel + kel.lbm.core + kel.lbm.swarm + ]; + + # No checkPhase, and that is a limitation rather than a choice. The nix build + # sandbox runs in a network namespace holding only a loopback interface, and + # ICE finds no usable candidate there -- libdatachannel reports "Failed to + # gather local ICE candidates" and negotiation never completes. Verified: the + # same two binaries negotiate and register fine outside the sandbox. + # + # The end to end check therefore ships as an executable instead, so it is one + # command rather than a procedure: + # + # nix-build -A poc.web_swarm.native + # ./result/bin/kel_lbm_web_swarm_check + # + # The protocol itself is covered natively and in-sandbox by kel.lbm.swarm. + doCheck = false; + + preferLocalBuild = true; + + outputs = [ "out" ]; +} diff --git a/poc/web_swarm/SConscript b/poc/web_swarm/SConscript deleted file mode 100644 index 64d0b79..0000000 --- a/poc/web_swarm/SConscript +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/false - -import os -import os.path -import glob - - -Import('env') - -dir_path = Dir('.').abspath - -# Environment for base library -examples_env = env.Clone(); -examples_env['CXX'] = 'syclcc-clang'; -examples_env['CXXFLAGS'] += ['-O3']; - -examples_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) -examples_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) - -env.sources += examples_env.sources; -env.headers += examples_env.headers; - -# Cavity2D -examples_objects = []; -examples_env.add_source_files(examples_objects, ['sim.cpp'], shared=False); -examples_env.poiseulle_2d_gpu = examples_env.Program('#bin/kel_lbm_web_swarm', [examples_objects]); - -# Set Alias -env.examples = [ - examples_env.poiseulle_2d_gpu -]; -env.Alias('examples', env.examples); -env.targets += ['examples']; -env.Install('$prefix/bin/', env.examples); diff --git a/poc/web_swarm/SConstruct b/poc/web_swarm/SConstruct index 0611b67..9d43609 100644 --- a/poc/web_swarm/SConstruct +++ b/poc/web_swarm/SConstruct @@ -43,23 +43,20 @@ env_vars.Add('prefix', validator=isAbsolutePath ) -env_vars.Add('build_examples', - help='If examples should be built', - default="true" -) - +# No forstio. The orchestrator is the one native binary here that does not use +# it: libdatachannel brings its own threads and event handling, and forstio has +# no WebSocket or WebRTC to contribute. env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], - CPPDEFINES=['SAW_UNIX'], + CPPDEFINES=[], CXXFLAGS=[ '-std=c++20', '-g', + '-O2', '-Wall', '-Wextra' ], LIBS=[ - 'forstio-core', - 'forstio-async', - 'forstio-io' + 'datachannel' ] ); env.__class__.add_source_files = add_kel_source_files @@ -71,11 +68,16 @@ env.sources = []; env.headers = []; env.targets = []; +env.check_script = File('#check.sh'); + Export('env') -SConscript('SConscript') +SConscript('orchestrator/c++/SConscript') +SConscript('worker_native/c++/SConscript') env.Alias('cdb', env.cdb); env.Alias('all', [env.targets]); env.Default('all'); +env.InstallAs('$prefix/bin/kel_lbm_web_swarm_check', env.check_script); + env.Alias('install', '$prefix') diff --git a/poc/web_swarm/check.sh b/poc/web_swarm/check.sh new file mode 100755 index 0000000..4cf6b87 --- /dev/null +++ b/poc/web_swarm/check.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# End to end check: does a native worker register with the orchestrator over a +# real WebRTC data channel? +# +# This is not a nix checkPhase, and that is not an oversight. The nix build +# sandbox runs in a network namespace holding nothing but a loopback interface, +# and ICE finds no usable candidate there -- libdatachannel reports "Failed to +# gather local ICE candidates" and negotiation never completes. The binaries are +# fine; there is simply nothing to gather. So the check lives here and is run +# outside the sandbox: +# +# nix-build -A poc.web_swarm.native +# ./result/bin/kel_lbm_web_swarm_check +# +# Exits 0 on success, non-zero with both logs printed otherwise. + +set -u + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +orchestrator="$here/kel_lbm_orchestrator" +worker="$here/kel_lbm_worker_native" + +for binary in "$orchestrator" "$worker"; do + if [ ! -x "$binary" ]; then + echo "missing executable: $binary" >&2 + exit 1 + fi +done + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT + +orch_log="$workdir/orch.log" +worker_log="$workdir/worker.log" + +# Port 0 asks the OS for a free one, so the port is read back rather than +# assumed -- otherwise two runs on one machine collide. +# Fast heartbeat so three beats land well inside the worker timeout. +"$orchestrator" --port 0 --heartbeat 300 --monitor 0 > "$orch_log" 2>&1 & +orch_pid=$! +trap 'kill $orch_pid 2>/dev/null; rm -rf "$workdir"' EXIT + +port="" +for _ in $(seq 1 100); do + port="$(sed -n 's|.*ws://0\.0\.0\.0:\([0-9]\+\).*|\1|p' "$orch_log")" + [ -n "$port" ] && break + if ! kill -0 $orch_pid 2>/dev/null; then + echo "orchestrator exited before reporting a port:" >&2 + cat "$orch_log" >&2 + exit 1 + fi + sleep 0.1 +done + +if [ -z "$port" ]; then + echo "orchestrator never reported a port:" >&2 + cat "$orch_log" >&2 + exit 1 +fi + +echo "orchestrator listening on $port" + +"$worker" \ + --url "ws://127.0.0.1:$port/" \ + --id check-native \ + --once \ + --pongs 3 \ + --timeout 20000 > "$worker_log" 2>&1 +worker_rc=$? + +kill $orch_pid 2>/dev/null +wait $orch_pid 2>/dev/null + +fail(){ + echo "FAIL: $1" >&2 + echo "--- orchestrator ---" >&2; cat "$orch_log" >&2 + echo "--- worker ---" >&2; cat "$worker_log" >&2 + exit 1 +} + +[ $worker_rc -eq 0 ] || fail "worker exited $worker_rc" + +# Both sides have to agree the channel came up, not just the worker. +grep -q "channel open" "$orch_log" \ + || fail "orchestrator never saw the control channel open" +grep -q "control channel open" "$worker_log" \ + || fail "worker never saw the control channel open" + +# The hello has to have been accepted, as a native worker, and answered. +grep -q "registered check-native (native)" "$orch_log" \ + || fail "orchestrator never registered the worker as native" +grep -q "<- welcome" "$worker_log" \ + || fail "worker never received its welcome" + +# Liveness, not just registration: the heartbeat has to be running and the +# worker has to be answering it. +grep -q "answered 3 ping(s)" "$worker_log" \ + || fail "worker did not answer three heartbeats" + +echo "PASS: native worker registered over a real data channel" diff --git a/poc/web_swarm/default.nix b/poc/web_swarm/default.nix index 570debf..e9ea108 100644 --- a/poc/web_swarm/default.nix +++ b/poc/web_swarm/default.nix @@ -1,6 +1,20 @@ { pkgs ? import <nixpkgs> {} +, stdenv +, pname +, version +, kel }: { - ui = pkgs.callPackage ./ui/.nix/derivation.nix {}; + # Orchestrator and native worker share a SConstruct and are checked against + # each other, so they are one derivation. + native = pkgs.callPackage ./.nix/derivation.nix { + inherit stdenv pname version kel; + }; + + # The browser worker is separate: different toolchain, and it cannot be + # tested in a check phase because a check phase has no browser. + worker = pkgs.callPackage ./worker/.nix/derivation.nix { + inherit stdenv pname version kel; + }; } diff --git a/poc/web_swarm/orchestrater/c++/main.cpp b/poc/web_swarm/orchestrater/c++/main.cpp deleted file mode 100644 index 891095e..0000000 --- a/poc/web_swarm/orchestrater/c++/main.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include <forstio/codec/args.hpp> - -namespace kel { -namespace sch { -} - -kel::lbm::error_or<void> kmain(int argc, char** argv){ - kel::lbm::data<sch::Args> args; - // Decode the args - { - kel::lbm::data<sch::Args,encode::Args> arge; - auto eov = saw::parse_args<sch::Args>(arge,args); - if(eov.is_error()){ - return eov; - } - } - return kel::lbm::make_void(); -} - -} - -int main(int argc, char** argv){ - auto eov = kel::kmain(argc,argv); - if(eov.is_error()){ - auto& err = eov.get_error(); - } - return 0; -} diff --git a/poc/web_swarm/orchestrator/c++/SConscript b/poc/web_swarm/orchestrator/c++/SConscript new file mode 100644 index 0000000..c4a7608 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/SConscript @@ -0,0 +1,26 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +orch_env = env.Clone(); + +orch_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +orch_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += orch_env.sources; +env.headers += orch_env.headers; + +objects = [] +orch_env.add_source_files(objects, orch_env.sources, shared=False); +env.orchestrator = orch_env.Program('#bin/kel_lbm_orchestrator', [objects]); + +env.targets += [env.orchestrator]; + +env.Install('$prefix/bin/', env.orchestrator); diff --git a/poc/web_swarm/orchestrator/c++/control.cpp b/poc/web_swarm/orchestrator/c++/control.cpp new file mode 100644 index 0000000..6e2d7b7 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/control.cpp @@ -0,0 +1,276 @@ +#include "control.hpp" + +#include <kel/lbm/swarm/interpret.hpp> +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <chrono> +#include <iomanip> +#include <sstream> +#include <utility> + +namespace kel { +namespace lbm { +namespace swarm { + +namespace { + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +} // namespace + +control::control(orchestrator &orch, control_config cfg) + : orch_{orch}, cfg_{std::move(cfg)}, registry_{cfg_.limits} {} + +control::~control(){ + stop_heartbeat(); +} + +void control::on_event(std::function<void(const std::string &)> handler){ + on_event_ = std::move(handler); +} + +void control::emit(const std::string &line){ + if(on_event_){ + on_event_(line); + } +} + +void control::attach(){ + orch_.on_channel_open([this](const std::shared_ptr<session> &sess){ + registry_.add_session(sess->id, now_ms()); + emit("[" + sess->id + "] channel open, awaiting hello"); + }); + + orch_.on_channel_closed([this](const std::shared_ptr<session> &sess){ + auto rec = registry_.remove(sess->id); + if(rec){ + emit("[" + sess->id + "] disconnected (" + + (rec->worker_id.empty() ? "unregistered" : rec->worker_id) + ")"); + } + }); + + orch_.on_message([this](const std::shared_ptr<session> &sess, + const std::string &text){ + handle_message(sess, text); + }); +} + +void control::handle_message(const std::shared_ptr<session> &sess, + const std::string &text){ + auto eo_hdr = peek_header(text); + if(eo_hdr.is_error()){ + emit("[" + sess->id + "] malformed control message, ignoring"); + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + const uint64_t seq = eo_hdr.get_value().get<"seq">().get(); + + if(type == msg::hello){ + handle_hello(sess, text); + return; + } + + if(type == msg::pong){ + auto eo = decode_message<sch::MsgPong>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed pong, ignoring"); + return; + } + registry_.record_pong(sess->id, seq, + eo.get_value().get<"sent_at_ms">().get(), + eo.get_value().get<"live">(), now_ms()); + return; + } + + if(type == msg::ready){ + auto eo = decode_message<sch::MsgReady>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed ready, ignoring"); + return; + } + registry_.mark_ready(sess->id, eo.get_value().get<"live">(), now_ms()); + emit("[" + sess->id + "] ready"); + return; + } + + if(type == msg::status){ + auto eo = decode_message<sch::MsgStatus>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed status, ignoring"); + return; + } + const auto &live = eo.get_value().get<"live">(); + registry_.update_live(sess->id, live, now_ms()); + + const std::string last_error = live.get<"last_error">().stl_string(); + if(not last_error.empty()){ + emit("[" + sess->id + "] reports error: " + last_error); + } + return; + } + + emit("[" + sess->id + "] unknown message type '" + type + "', ignoring"); +} + +void control::handle_hello(const std::shared_ptr<session> &sess, + const std::string &text){ + // Routing pass: the common capability fields are enough to learn the kind, + // and the backend section is skipped because it is not named here. + auto eo_route = decode_message<sch::MsgHello>(text); + if(eo_route.is_error()){ + emit("[" + sess->id + "] hello missing its common fields, rejecting"); + return; + } + + const auto &common = eo_route.get_value().get<"capability">(); + const std::string kind = common.get<"kind">().stl_string(); + const std::string worker_id = common.get<"worker_id">().stl_string(); + const std::string their_version = common.get<"schema_version">().stl_string(); + + if(their_version != schema_version){ + // Caught deliberately rather than suffered as a decode failure later. + emit("[" + sess->id + "] schema version mismatch: worker says '" + + their_version + "', orchestrator is '" + std::string{schema_version} + + "' -- rejecting"); + return; + } + + uint64_t cells = 0u; + std::string summary; + + if(kind == wkind::webgl2){ + auto eo = decode_message<sch::MsgHelloWebGl2>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] hello claims webgl2 but has no usable " + "webgl2 section, rejecting"); + return; + } + const auto &gl = eo.get_value().get<"capability">().get<"webgl2">(); + cells = webgl2_capacity_cells(gl); + summary = webgl2_summary(gl); + + } else if(kind == wkind::native){ + auto eo = decode_message<sch::MsgHelloNative>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] hello claims native but has no usable " + "native section, rejecting"); + return; + } + const auto &nat = eo.get_value().get<"capability">().get<"native">(); + cells = native_capacity_cells(nat); + summary = native_summary(nat); + + } else { + // Identifiable but unusable, which is a better outcome than a zeroed + // registration and is exactly what the routing decode is for. + emit("[" + sess->id + "] unsupported worker kind '" + kind + + "', rejecting"); + return; + } + + const uint64_t t = now_ms(); + if(not registry_.mark_registered(sess->id, worker_id, kind, cells, + summary, t)){ + return; + } + + auto welcome = make_message<sch::MsgWelcome>(msg::welcome, sess->id, 0u); + welcome.get<"heartbeat_ms">().set(cfg_.heartbeat_ms); + welcome.get<"schema_version">().set(std::string{schema_version}); + + auto eo_out = encode_message<sch::MsgWelcome>(welcome); + if(eo_out.is_error()){ + return; + } + orch_.send(sess, eo_out.get_value()); + + std::ostringstream line; + line << "[" << sess->id << "] registered " << worker_id + << " (" << kind << ") -- " << summary; + emit(line.str()); +} + +void control::start_heartbeat(){ + if(running_.exchange(true)){ + return; + } + ticker_ = std::thread{[this](){ heartbeat_loop(); }}; +} + +void control::stop_heartbeat(){ + if(not running_.exchange(false)){ + return; + } + tick_cv_.notify_all(); + if(ticker_.joinable()){ + ticker_.join(); + } +} + +void control::heartbeat_loop(){ + while(running_.load()){ + { + std::unique_lock<std::mutex> lock{tick_mutex_}; + tick_cv_.wait_for(lock, std::chrono::milliseconds{cfg_.heartbeat_ms}, + [this](){ return not running_.load(); }); + } + if(not running_.load()){ + break; + } + + const uint64_t t = now_ms(); + + // The registry decides; this loop only carries out the decisions. In + // particular nothing is sent while its lock is held. + auto result = registry_.tick(t); + + for(const auto &id : result.newly_stale){ + emit("[" + id + "] stale, no response to the last " + + std::to_string(cfg_.limits.stale_after) + " pings"); + } + + for(const auto &rec : result.dropped){ + emit("[" + rec.session_id + "] dropped after " + + std::to_string(cfg_.limits.drop_after) + " missed pings (" + + (rec.worker_id.empty() ? "unregistered" : rec.worker_id) + ")"); + + // Tear the transport down too, or the session lingers with nothing + // watching it. + if(auto sess = orch_.find_session(rec.session_id)){ + if(sess->dc){ + sess->dc->close(); + } + if(sess->pc){ + sess->pc->close(); + } + } + } + + for(const auto &target : result.to_ping){ + auto sess = orch_.find_session(target.session_id); + if(not sess){ + continue; + } + + auto ping = make_message<sch::MsgPing>(msg::ping, target.session_id, + target.seq); + ping.get<"sent_at_ms">().set(t); + + auto eo = encode_message<sch::MsgPing>(ping); + if(eo.is_error()){ + continue; + } + orch_.send(sess, eo.get_value()); + } + } +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/control.hpp b/poc/web_swarm/orchestrator/c++/control.hpp new file mode 100644 index 0000000..9365ef9 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/control.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "orchestrator.hpp" +#include <kel/lbm/swarm/registry.hpp> + +#include <atomic> +#include <condition_variable> +#include <cstdint> +#include <functional> +#include <mutex> +#include <string> +#include <thread> + +namespace kel { +namespace lbm { +namespace swarm { + +struct control_config { + uint64_t heartbeat_ms = 2000u; + registry::thresholds limits; +}; + +/** + * The control protocol on top of the transport. + * + * Kept apart from `orchestrator` so each can be reasoned about alone: the + * transport knows nothing about hellos, and this knows nothing about ICE. + * + * Owns the heartbeat thread. libdatachannel provides no timers, so liveness is + * a plain ticker -- which is fine, since the work it does per beat is a lock, a + * vector copy and some sends. + */ +class control { +public: + control(orchestrator &orch, control_config cfg); + ~control(); + + control(const control &) = delete; + control &operator=(const control &) = delete; + + /// Installs the handlers on the orchestrator. Call before listen(). + void attach(); + + void start_heartbeat(); + void stop_heartbeat(); + + registry &workers() { return registry_; } + + /// Emitted for anything worth a line in the log. + void on_event(std::function<void(const std::string &)> handler); + +private: + void handle_message(const std::shared_ptr<session> &sess, + const std::string &text); + void handle_hello(const std::shared_ptr<session> &sess, + const std::string &text); + void heartbeat_loop(); + void emit(const std::string &line); + + orchestrator &orch_; + control_config cfg_; + registry registry_; + + std::thread ticker_; + std::mutex tick_mutex_; + std::condition_variable tick_cv_; + std::atomic<bool> running_{false}; + + std::function<void(const std::string &)> on_event_; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/main.cpp b/poc/web_swarm/orchestrator/c++/main.cpp new file mode 100644 index 0000000..4400a0d --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/main.cpp @@ -0,0 +1,205 @@ +#include "control.hpp" +#include "orchestrator.hpp" + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <csignal> +#include <cstdlib> +#include <iostream> +#include <mutex> +#include <string> + +/** + * The swarm orchestrator. + * + * Accepts workers over a WebSocket, negotiates a peer connection, opens a + * control data channel, registers whatever capability the worker reports, and + * keeps the connection under a heartbeat so liveness is a fact rather than an + * assumption. + * + * There is no event loop of our own. libdatachannel runs its own threads and + * the heartbeat runs on another, so main's whole job is to start things, print + * what happens, and wait for a signal. + */ +namespace { + +std::mutex g_wait_mutex; +std::condition_variable g_wait_cv; +std::atomic<bool> g_running{true}; +std::atomic<bool> g_dump_requested{false}; + +/** + * Signal handlers may only touch lock free atomics, so these set a flag and + * poke the condition variable. The notify is outside what the standard blesses + * in a handler, but the predicate is re-checked on a timed wakeup, so a lost + * notify costs at most the poll interval rather than a hang. + */ +extern "C" void handle_stop(int){ + g_running.store(false); + g_wait_cv.notify_all(); +} + +extern "C" void handle_dump(int){ + g_dump_requested.store(true); + g_wait_cv.notify_all(); +} + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +void print_usage(const char *argv0){ + std::cout + << "Usage: " << argv0 << " [options]\n" + << "\n" + << " --port <n> Port for the signalling WebSocket\n" + << " (default 8080). 0 asks the OS for a free one.\n" + << " --stun <url> STUN server, e.g.\n" + << " stun:stun.l.google.com:19302. Not needed on\n" + << " localhost or a LAN.\n" + << " --heartbeat <ms> Ping interval (default 2000).\n" + << " --stale-after <n> Missed pings before a worker is called stale\n" + << " (default 3).\n" + << " --drop-after <n> Missed pings before it is dropped (default 6).\n" + << " --monitor <ms> Print the worker table this often. 0 disables\n" + << " (default 5000). Nothing is printed while no\n" + << " workers are connected.\n" + << " --help This text.\n" + << "\n" + << "SIGUSR1 prints the worker table immediately.\n" + << "\n" + << "The worker page is not served from here -- a WebSocket server is\n" + << "not an HTTP server. Serve it separately, e.g.\n" + << " python3 -m http.server 8000\n"; +} + +} // namespace + +int main(int argc, char **argv){ + using namespace kel::lbm; + + swarm::orchestrator_config ocfg; + swarm::control_config ccfg; + uint64_t monitor_ms = 5000u; + + for(int i = 1; i < argc; ++i){ + const std::string arg{argv[i]}; + + if(arg == "--help" or arg == "-h"){ + print_usage(argv[0]); + return 0; + } + if(arg == "--port" and (i + 1) < argc){ + ocfg.port = static_cast<uint16_t>(std::strtoul(argv[++i], nullptr, 10)); + continue; + } + if(arg == "--stun" and (i + 1) < argc){ + ocfg.stun_url = argv[++i]; + continue; + } + if(arg == "--heartbeat" and (i + 1) < argc){ + ccfg.heartbeat_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--stale-after" and (i + 1) < argc){ + ccfg.limits.stale_after = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--drop-after" and (i + 1) < argc){ + ccfg.limits.drop_after = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--monitor" and (i + 1) < argc){ + monitor_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + + std::cerr << "Unknown or incomplete option: " << arg << "\n\n"; + print_usage(argv[0]); + return 2; + } + + if(ccfg.heartbeat_ms == 0u){ + std::cerr << "--heartbeat must be greater than zero\n"; + return 2; + } + if(ccfg.limits.drop_after < ccfg.limits.stale_after){ + // Otherwise a worker is dropped before it can ever be reported stale, + // which makes the stale state unreachable and the log misleading. + std::cerr << "--drop-after must be at least --stale-after\n"; + return 2; + } + + std::signal(SIGINT, handle_stop); + std::signal(SIGTERM, handle_stop); + std::signal(SIGUSR1, handle_dump); + + swarm::orchestrator orch{ocfg}; + swarm::control ctl{orch, ccfg}; + + ctl.on_event([](const std::string &line){ + std::cout << line << std::endl; + }); + + ctl.attach(); + + uint16_t port = 0u; + try { + port = orch.listen(); + } catch (const std::exception &e){ + std::cerr << "[Error] Could not listen: " << e.what() << "\n"; + return 1; + } + + ctl.start_heartbeat(); + + std::cout << "orchestrator listening on ws://0.0.0.0:" << port << "\n" + << "heartbeat " << ccfg.heartbeat_ms << "ms, stale after " + << ccfg.limits.stale_after << " misses, drop after " + << ccfg.limits.drop_after << "\n" + << "ctrl-c to stop, SIGUSR1 for the worker table" << std::endl; + + uint64_t last_monitor = now_ms(); + + { + std::unique_lock<std::mutex> lock{g_wait_mutex}; + while(g_running.load()){ + g_wait_cv.wait_for(lock, std::chrono::milliseconds{250}, + [](){ return not g_running.load() + or g_dump_requested.load(); }); + + const bool asked = g_dump_requested.exchange(false); + const uint64_t t = now_ms(); + const bool due = (monitor_ms > 0u) and ((t - last_monitor) >= monitor_ms); + + if(not g_running.load()){ + break; + } + if(not asked and not due){ + continue; + } + + last_monitor = t; + + auto snapshot = ctl.workers().snapshot(); + // An idle orchestrator should be quiet; a periodic table of + // nothing is just log noise. An explicit SIGUSR1 always prints, + // because then someone is actually asking. + if(snapshot.empty() and not asked){ + continue; + } + + std::cout << "\n" << snapshot.size() << " worker(s):\n" + << swarm::render_table(snapshot, t) << std::endl; + } + } + + std::cout << "\nstopping" << std::endl; + ctl.stop_heartbeat(); + orch.stop(); + + return 0; +} diff --git a/poc/web_swarm/orchestrator/c++/orchestrator.cpp b/poc/web_swarm/orchestrator/c++/orchestrator.cpp new file mode 100644 index 0000000..68ed8da --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/orchestrator.cpp @@ -0,0 +1,339 @@ +#include "orchestrator.hpp" + +#include <kel/lbm/swarm/signal.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <iostream> +#include <utility> + +namespace kel { +namespace lbm { +namespace swarm { + +namespace { + +/** + * libdatachannel delivers either text or binary. The control and signalling + * protocols are both JSON text, so binary is simply not ours and is dropped. + */ +std::string text_of(const rtc::message_variant &msg){ + if(std::holds_alternative<rtc::string>(msg)){ + return std::get<rtc::string>(msg); + } + return {}; +} + +} // namespace + +orchestrator::orchestrator(orchestrator_config cfg) : cfg_{std::move(cfg)} {} + +orchestrator::~orchestrator(){ + stop(); +} + +void orchestrator::on_message(message_handler handler){ + on_message_ = std::move(handler); +} + +void orchestrator::on_channel_open( + std::function<void(const std::shared_ptr<session> &)> handler){ + on_open_ = std::move(handler); +} + +void orchestrator::on_channel_closed( + std::function<void(const std::shared_ptr<session> &)> handler){ + on_closed_ = std::move(handler); +} + +std::string orchestrator::next_session_id(){ + // Called under the lock. + return "w" + std::to_string(++next_id_); +} + +uint16_t orchestrator::listen(){ + rtc::WebSocketServerConfiguration wscfg; + wscfg.port = cfg_.port; + wscfg.enableTls = false; + + server_ = std::make_unique<rtc::WebSocketServer>(std::move(wscfg)); + server_->onClient([this](std::shared_ptr<rtc::WebSocket> ws){ + accept_client(std::move(ws)); + }); + + return server_->port(); +} + +void orchestrator::accept_client(std::shared_ptr<rtc::WebSocket> ws){ + auto sess = std::make_shared<session>(); + sess->ws = std::move(ws); + + { + std::lock_guard<std::mutex> lock{mutex_}; + sess->id = next_session_id(); + sessions_.emplace(sess->id, sess); + } + + // Callbacks capture weakly. A worker that drops mid-negotiation would + // otherwise keep its own session alive through the capture and never be + // collected. + std::weak_ptr<session> weak = sess; + + sess->ws->onMessage([this, weak](rtc::message_variant msg){ + auto s = weak.lock(); + if(not s){ + return; + } + const std::string text = text_of(msg); + if(text.empty()){ + return; + } + handle_signal(s, text); + }); + + sess->ws->onClosed([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + // The data channel may already be gone; dropping the session is + // idempotent so the two paths cannot double-remove. + drop_session(s->id); + }); + + begin_negotiation(sess); +} + +void orchestrator::begin_negotiation(const std::shared_ptr<session> &sess){ + rtc::Configuration rtccfg; + if(not cfg_.stun_url.empty()){ + rtccfg.iceServers.emplace_back(cfg_.stun_url); + } + + sess->pc = std::make_shared<rtc::PeerConnection>(rtccfg); + + std::weak_ptr<session> weak = sess; + + sess->pc->onLocalDescription([weak](rtc::Description desc){ + auto s = weak.lock(); + if(not s or not s->ws or not s->ws->isOpen()){ + return; + } + + data<sch::SignalDescription> out; + out.get<"type">().set(desc.typeString()); + out.get<"sdp">().set(std::string{desc}); + + auto eo_text = encode_message<sch::SignalDescription>(out); + if(eo_text.is_error()){ + return; + } + s->ws->send(eo_text.get_value()); + }); + + sess->pc->onLocalCandidate([weak](rtc::Candidate cand){ + auto s = weak.lock(); + if(not s or not s->ws or not s->ws->isOpen()){ + return; + } + + data<sch::SignalCandidate> out; + out.get<"type">().set(std::string{sig::candidate}); + out.get<"candidate">().set(cand.candidate()); + out.get<"mid">().set(cand.mid()); + + auto eo_text = encode_message<sch::SignalCandidate>(out); + if(eo_text.is_error()){ + return; + } + s->ws->send(eo_text.get_value()); + }); + + sess->pc->onStateChange([this, weak](rtc::PeerConnection::State state){ + auto s = weak.lock(); + if(not s){ + return; + } + if(state == rtc::PeerConnection::State::Closed or + state == rtc::PeerConnection::State::Failed){ + drop_session(s->id); + } + }); + + /** + * Creating the channel here is what triggers negotiation: with automatic + * negotiation on, the offer is generated because there is now something to + * offer. Doing it from the orchestrator rather than waiting for the worker + * also fixes the ordering -- the channel is in the offer, so the worker + * never has to guess whether it exists yet. + */ + sess->dc = sess->pc->createDataChannel("control"); + + sess->dc->onOpen([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + if(on_open_){ + on_open_(s); + } + }); + + sess->dc->onMessage([this, weak](rtc::message_variant msg){ + auto s = weak.lock(); + if(not s){ + return; + } + const std::string text = text_of(msg); + if(text.empty()){ + return; + } + if(on_message_){ + on_message_(s, text); + } + }); + + sess->dc->onClosed([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + drop_session(s->id); + }); +} + +void orchestrator::handle_signal(const std::shared_ptr<session> &sess, + const std::string &text){ + // Two stage decode, same as the control protocol: learn the type without + // committing to a payload shape. + auto eo_hdr = decode_message<sch::SignalHeader>(text); + if(eo_hdr.is_error()){ + std::cerr << "[" << sess->id << "] malformed signalling message\n"; + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == sig::answer or type == sig::offer){ + auto eo = decode_message<sch::SignalDescription>(text); + if(eo.is_error()){ + std::cerr << "[" << sess->id << "] bad " << type << "\n"; + return; + } + try { + sess->pc->setRemoteDescription( + rtc::Description{eo.get_value().get<"sdp">().stl_string(), type}); + } catch (const std::exception &e){ + std::cerr << "[" << sess->id << "] setRemoteDescription: " + << e.what() << "\n"; + } + return; + } + + if(type == sig::candidate){ + auto eo = decode_message<sch::SignalCandidate>(text); + if(eo.is_error()){ + std::cerr << "[" << sess->id << "] bad candidate\n"; + return; + } + try { + sess->pc->addRemoteCandidate( + rtc::Candidate{eo.get_value().get<"candidate">().stl_string(), + eo.get_value().get<"mid">().stl_string()}); + } catch (const std::exception &e){ + std::cerr << "[" << sess->id << "] addRemoteCandidate: " + << e.what() << "\n"; + } + return; + } + + std::cerr << "[" << sess->id << "] unknown signalling type: " << type << "\n"; +} + +void orchestrator::drop_session(const std::string &id){ + std::shared_ptr<session> sess; + + { + std::lock_guard<std::mutex> lock{mutex_}; + auto iter = sessions_.find(id); + if(iter == sessions_.end()){ + // Already dropped. Several teardown paths converge here -- socket + // closed, channel closed, peer connection failed -- and whichever + // arrives first wins. + return; + } + sess = iter->second; + sessions_.erase(iter); + } + + // Notify outside the lock: the handler may well want to send to other + // sessions, and holding the registry lock across user code is how this + // kind of thing deadlocks later. + if(on_closed_){ + on_closed_(sess); + } +} + +bool orchestrator::send(const std::shared_ptr<session> &sess, + const std::string &text){ + if(not sess or not sess->dc or not sess->dc->isOpen()){ + return false; + } + return sess->dc->send(text); +} + +std::shared_ptr<session> orchestrator::find_session(const std::string &id) const { + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = sessions_.find(id); + if(iter == sessions_.end()){ + return nullptr; + } + + return iter->second; +} + +std::size_t orchestrator::session_count() const { + std::lock_guard<std::mutex> lock{mutex_}; + return sessions_.size(); +} + +void orchestrator::for_each_session( + const std::function<void(const std::shared_ptr<session> &)> &fn) const { + std::lock_guard<std::mutex> lock{mutex_}; + for(const auto &entry : sessions_){ + fn(entry.second); + } +} + +void orchestrator::stop(){ + std::map<std::string, std::shared_ptr<session>> taken; + + { + std::lock_guard<std::mutex> lock{mutex_}; + taken.swap(sessions_); + } + + // Close outside the lock. Closing a peer connection runs callbacks, and + // some of those callbacks want the lock. + for(auto &entry : taken){ + auto &sess = entry.second; + if(sess->dc){ + sess->dc->close(); + } + if(sess->pc){ + sess->pc->close(); + } + if(sess->ws){ + sess->ws->close(); + } + } + + if(server_){ + server_->stop(); + server_.reset(); + } +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/orchestrator.hpp b/poc/web_swarm/orchestrator/c++/orchestrator.hpp new file mode 100644 index 0000000..83ce835 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/orchestrator.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include "session.hpp" + +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <mutex> +#include <string> +#include <string_view> + +namespace kel { +namespace lbm { +namespace swarm { + +struct orchestrator_config { + uint16_t port = 8080u; + + /** + * STUN is not needed on localhost or a LAN -- host candidates are enough, + * and they are gathered without any server. The slot is here so NAT + * traversal is a configuration change later rather than a code change. + */ + std::string stun_url; +}; + +/** + * Accepts workers and holds their connections open. + * + * This is the first code in the repository with concurrency in it, so the rule + * is worth stating rather than inferring: libdatachannel owns the threads and + * delivers every callback on one of them. The only mutable state the + * orchestrator itself owns is the session map, and it is guarded by one mutex. + * Nothing else here is shared. Resist growing that. + * + * A deliberate non-feature: no static file serving. The page has to come from + * somewhere else (python3 -m http.server over the nix output is fine for + * development), because a WebSocket server is not an HTTP server and pretending + * otherwise would mean carrying a second protocol for no benefit. + */ +class orchestrator { +public: + /** + * Called with the session and the decoded text of each control message. + * Handling lives outside this class so the transport can be tested without + * a protocol and the protocol without a transport. + */ + using message_handler = + std::function<void(const std::shared_ptr<session> &, const std::string &)>; + + explicit orchestrator(orchestrator_config cfg); + ~orchestrator(); + + orchestrator(const orchestrator &) = delete; + orchestrator &operator=(const orchestrator &) = delete; + + /// Starts listening. Returns the port actually bound, which differs from + /// the requested one when the config asked for 0. + uint16_t listen(); + + void on_message(message_handler handler); + + /// Invoked once a session's control channel is open and usable. + void on_channel_open(std::function<void(const std::shared_ptr<session> &)> handler); + + /// Invoked when a session goes away, however it goes away. + void on_channel_closed(std::function<void(const std::shared_ptr<session> &)> handler); + + /// Sends text on a session's control channel. False if it is not open. + bool send(const std::shared_ptr<session> &sess, const std::string &text); + + /// Number of sessions currently held. Takes the lock. + std::size_t session_count() const; + + /// Runs `fn` for each session under the lock. Keep it short. + void for_each_session( + const std::function<void(const std::shared_ptr<session> &)> &fn) const; + + /** + * Resolves a session id to something sendable, or null if it has gone. + * + * The registry deals in ids alone so it can stay transport free, which + * leaves this as the one place that maps an id back to a connection. + */ + std::shared_ptr<session> find_session(const std::string &id) const; + + void stop(); + +private: + void accept_client(std::shared_ptr<rtc::WebSocket> ws); + void begin_negotiation(const std::shared_ptr<session> &sess); + void handle_signal(const std::shared_ptr<session> &sess, const std::string &text); + void drop_session(const std::string &id); + + std::string next_session_id(); + + orchestrator_config cfg_; + + std::unique_ptr<rtc::WebSocketServer> server_; + + mutable std::mutex mutex_; + std::map<std::string, std::shared_ptr<session>> sessions_; + uint64_t next_id_ = 0u; + + message_handler on_message_; + std::function<void(const std::shared_ptr<session> &)> on_open_; + std::function<void(const std::shared_ptr<session> &)> on_closed_; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/session.hpp b/poc/web_swarm/orchestrator/c++/session.hpp new file mode 100644 index 0000000..cdc6be3 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/session.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include <rtc/rtc.hpp> + +#include <memory> +#include <mutex> +#include <string> + +namespace kel { +namespace lbm { +namespace swarm { + +/** + * One worker's connection, from the signalling socket through to the control + * channel. + * + * Lifetime is the reason this is a type rather than three locals. libdatachannel + * hands callbacks a reference to nothing in particular, and a PeerConnection + * dropped at the end of the accept handler takes the whole negotiation with it. + * The orchestrator therefore holds a shared_ptr per session and the callbacks + * capture a weak_ptr, so a worker that disconnects mid-negotiation tears down + * cleanly instead of leaving a callback pointing at freed state. + * + * Threading: every member here is touched from libdatachannel's threads, not + * from main. The handles themselves are set once during setup and then only + * read, and the mutable state behind them is libdatachannel's own problem. The + * orchestrator's own mutable state lives in the registry, behind its mutex. + */ +struct session { + /// Assigned by the orchestrator, not the worker. Stable for the connection. + std::string id; + + /// Signalling. Stays open after negotiation so renegotiation is possible. + std::shared_ptr<rtc::WebSocket> ws; + + std::shared_ptr<rtc::PeerConnection> pc; + + /** + * The control channel. Created by the orchestrator rather than the worker + * so that channel-open ordering is deterministic: the offer already + * contains it, so by the time the worker answers it knows the channel is + * coming and does not have to race onDataChannel against its own send. + */ + std::shared_ptr<rtc::DataChannel> dc; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/ui/.nix/derivation.nix b/poc/web_swarm/ui/.nix/derivation.nix deleted file mode 100644 index 8dcabe1..0000000 --- a/poc/web_swarm/ui/.nix/derivation.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ lib -, stdenv -, scons -, clang-tools -, forstio -, python3 -, pname -, version -, adaptive-cpp -, kel -}: - -stdenv.mkDerivation { - pname = pname + "-poc-" + "web_swarm_ui"; - inherit version; - src = ./..; - - nativeBuildInputs = [ - scons - clang-tools - python3 - ]; - - buildInputs = [ - forstio.core - forstio.async - forstio.codec - forstio.codec-unit - forstio.io - forstio.remote - forstio.remote-filesystem - forstio.codec-json - adaptive-cpp - kel.lbm.core - kel.lbm.sycl - ]; - - preferLocalBuild = true; - - outputs = [ "out" "dev" ]; -} diff --git a/poc/web_swarm/ui/html/index.html b/poc/web_swarm/ui/html/index.html deleted file mode 100644 index e69de29..0000000 --- a/poc/web_swarm/ui/html/index.html +++ /dev/null diff --git a/poc/web_swarm/worker/.nix/derivation.nix b/poc/web_swarm/worker/.nix/derivation.nix new file mode 100644 index 0000000..694d951 --- /dev/null +++ b/poc/web_swarm/worker/.nix/derivation.nix @@ -0,0 +1,80 @@ +{ lib +, stdenv +, scons +, clang-tools +, emscripten +, python3 +, nodejs +, pname +, version +, kel +}: + +stdenv.mkDerivation { + pname = "${pname}-web-swarm-worker"; + inherit version; + src = ./..; + + nativeBuildInputs = [ + scons + clang-tools + emscripten + python3 + nodejs + ]; + + # Deliberately empty. em++ is not a nix cc-wrapper, so it never reads + # NIX_CFLAGS_COMPILE and cannot consume buildInputs the way the native + # builds do. Include roots are handed to scons explicitly below, which also + # has the virtue of making the dependency visible rather than ambient. + buildInputs = [ ]; + + sconsFlags = [ + ("kel_include=" + + "${kel.lbm.core.dev}/include:" + + "${kel.lbm.swarm.dev}/include:" + + "${kel.lbm.webgl.dev}/include") + ]; + + # Same class of problem as the AdaptiveCpp HOME hack in + # modules/sycl/.nix/derivation.nix: a toolchain that insists on a writable + # cache inside a read-only build. Seed it from the cache the emscripten + # derivation already precompiled, or every build rebuilds the sysroot. + preBuild = '' + export HOME="$TMPDIR" + export EM_CACHE="$TMPDIR/em_cache" + cp -r ${emscripten}/share/emscripten/cache "$EM_CACHE" + chmod -R u+w "$EM_CACHE" + ''; + + # The real worker needs a browser. Its protocol half does not, so that part + # is built for node and run here; only the GL probe is left uncovered. + doCheck = true; + + checkPhase = '' + runHook preCheck + + node ./bin/kel_lbm_worker_selftest.js + + runHook postCheck + ''; + + postInstall = '' + cat > $out/share/kel-lbm/worker/README <<'EOT' + Serve this directory over HTTP and open index.html. + + python3 -m http.server 8000 + + file:// will not work: wasm is fetched and file:// is a null origin. + + The orchestrator is a separate process on its own port -- a WebSocket + server is not an HTTP server, so it does not serve this page. Start it + with kel_lbm_orchestrator and point the page at it, either in the form + on the page or with ?orchestrator=ws://host:port/ + EOT + ''; + + preferLocalBuild = true; + + outputs = [ "out" ]; +} diff --git a/poc/web_swarm/worker/SConstruct b/poc/web_swarm/worker/SConstruct new file mode 100644 index 0000000..ae0f068 --- /dev/null +++ b/poc/web_swarm/worker/SConstruct @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +# em++ emits a .js loader plus a .wasm sidecar, but SCons only learns about the +# target it was asked for, so the .wasm is invisible to the dependency graph: +# `scons -c` leaves it behind and an incremental build can reuse a stale one. +# +# Program's emitter is an EmitterProxy rather than a dict, so add_emitter() is +# not available on it. Declaring the sidecar explicitly is the supported route: +# SideEffect keeps parallel builds from racing on it, Clean makes -c remove it. +def em_program(env, target, objects): + js = env.Program(target, objects) + wasm = env.File(str(js[0])[:-3] + '.wasm') + env.SideEffect(wasm, js) + env.Clean(js, wasm) + return [js[0], wasm] + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +# em++ is not a nix cc-wrapper, so it never sees NIX_CFLAGS_COMPILE and cannot +# pick up buildInputs the way every other module here does. The derivation +# passes the include roots explicitly instead; see .nix/derivation.nix. +env_vars.Add('kel_include', + help='Colon separated include roots for kel-lbm headers', + default='' +) + +env=Environment(ENV=os.environ, variables=env_vars, + CPPPATH=[p for p in ARGUMENTS.get('kel_include', '').split(':') if p], + CPPDEFINES=[], + CXX = 'em++', + CC = 'emcc', + AR = 'emar', + RANLIB = 'emranlib', + LINK = 'em++', + CXXFLAGS=[ + '-std=c++20', + '-O2', + '-Wall', + '-Wextra', + # kel-lbm's string_literal provides operator""_sl as a string literal + # operator template, which is a GNU extension the library leans on + # deliberately. Without this it warns once per translation unit from a + # core header and drowns out warnings that are actually ours. + '-Wno-gnu-string-literal-operator-template' + ], + LINKFLAGS=[ + # WebGL2 only, and the GLES3 entry points resolved through + # getProcAddress so extension functions work. + '-sFULL_ES3=1', + '-sWASM=1', + '-sMODULARIZE=1', + '-sENVIRONMENT=web', + '-sALLOW_MEMORY_GROWTH=1', + '-sMIN_WEBGL_VERSION=2', + '-sMAX_WEBGL_VERSION=2', + # The page drives the module through these rather than through main. + '-sEXPORTED_FUNCTIONS=_main,_kel_worker_init,_kel_worker_hello,' + '_kel_worker_handle,_kel_worker_pongs,_kel_worker_budget_bytes', + '-sEXPORTED_RUNTIME_METHODS=ccall,cwrap,UTF8ToString' + ], + LIBS=[], + PROGSUFFIX='.js' +); +env.__class__.add_source_files = add_kel_source_files +env.__class__.em_program = em_program + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('c++/SConscript') +SConscript('selftest/SConscript') + +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/poc/web_swarm/worker/c++/SConscript b/poc/web_swarm/worker/c++/SConscript new file mode 100644 index 0000000..a678b7b --- /dev/null +++ b/poc/web_swarm/worker/c++/SConscript @@ -0,0 +1,32 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +worker_env = env.Clone(); + +worker_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +worker_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += worker_env.sources; +env.headers += worker_env.headers; + +objects = [] +worker_env.add_source_files(objects, worker_env.sources, shared=False); +env.worker = worker_env.em_program('#bin/kel_lbm_worker', [objects]); + +env.targets += [env.worker]; + +# The page and its glue live beside the module: index.html loads worker.js for +# the transport and kel_lbm_worker.js for the emscripten loader. +web_dir = Dir('#web').abspath +web_files = sorted(glob.glob(web_dir + "/*")) + +env.Install('$prefix/share/kel-lbm/worker/', env.worker); +env.Install('$prefix/share/kel-lbm/worker/', web_files); diff --git a/poc/web_swarm/worker/c++/protocol.hpp b/poc/web_swarm/worker/c++/protocol.hpp new file mode 100644 index 0000000..16d379f --- /dev/null +++ b/poc/web_swarm/worker/c++/protocol.hpp @@ -0,0 +1,160 @@ +#pragma once + +#include <kel/lbm/webgl/probe.hpp> + +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <cstdint> +#include <string> + +/** + * What the browser worker does, separated from how JavaScript calls it. + * + * worker.cpp is a thin extern "C" shim over this; keeping the behaviour in a + * header means the self test can drive it directly under node, without a + * browser and without going through ccall. The GL probe is the only part that + * genuinely needs a browser, and it fails cleanly when there is no context -- + * which is exactly the state node is in, so everything else stays exercisable. + */ +namespace kel { +namespace lbm { +namespace swarm { + +class worker_protocol { +public: + struct config { + std::string worker_id = "browser"; + std::string user_agent = "unknown"; + }; + + /** + * Runs the capability probe. + * + * Returns false when this device cannot run a solver. That is not a reason + * to stay silent: the worker still registers, so the orchestrator learns + * the device exists and why it is unusable, rather than never hearing from + * it at all. + */ + bool init(config cfg, uint64_t now_ms){ + cfg_ = std::move(cfg); + started_at_ms_ = now_ms; + + webgl::probe_config pcfg; + probe_ok_ = webgl::probe_webgl2(pcfg, capability_); + + return probe_ok_; + } + + /// The hello to send once the control channel opens. + std::string hello(uint64_t now_ms) const { + auto message = make_message<sch::MsgHelloWebGl2>( + msg::hello, cfg_.worker_id, 0u); + + auto &cap = message.get<"capability">(); + cap.get<"worker_id">().set(cfg_.worker_id); + cap.get<"kind">().set(std::string{wkind::webgl2}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(now_ms); + cap.get<"user_agent">().set(cfg_.user_agent); + cap.get<"webgl2">() = capability_; + + auto eo = encode_message<sch::MsgHelloWebGl2>(message); + if(eo.is_error()){ + return {}; + } + return eo.get_value(); + } + + /** + * Handles one control message, returning the reply or an empty string. + * + * The transport never has to know that a ping wants a pong -- only that a + * non-empty return goes back down the channel. + */ + std::string handle(const std::string &text, uint64_t now_ms){ + auto eo_hdr = peek_header(text); + if(eo_hdr.is_error()){ + return {}; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == msg::ping){ + return handle_ping(text, now_ms); + } + if(type == msg::welcome){ + return handle_welcome(now_ms); + } + + // Anything else is not ours to answer. + return {}; + } + + uint64_t pongs() const { return pongs_; } + bool probe_ok() const { return probe_ok_; } + + const data<sch::WebGl2Capability> &capability() const { return capability_; } + +private: + data<sch::WorkerLive> make_live(uint64_t now_ms) const { + data<sch::WorkerLive> live; + live.get<"state">().set(std::string{ + probe_ok_ ? wstate::ready : wstate::error}); + live.get<"uptime_ms">().set( + (now_ms > started_at_ms_) ? (now_ms - started_at_ms_) : 0u); + // Nothing is being simulated yet; the field exists so the monitor does + // not need a protocol change once something is. + live.get<"steps_done">().set(0u); + live.get<"last_error">().set(std::string{""}); + return live; + } + + std::string handle_ping(const std::string &text, uint64_t now_ms){ + auto eo = decode_message<sch::MsgPing>(text); + if(eo.is_error()){ + return {}; + } + + auto pong = make_message<sch::MsgPong>( + msg::pong, cfg_.worker_id, eo.get_value().get<"seq">().get()); + + // Echo the orchestrator's own timestamp so it can measure the round + // trip against its clock. The two machines never have to agree on what + // time it is. + pong.get<"sent_at_ms">().set(eo.get_value().get<"sent_at_ms">().get()); + pong.get<"worker_now_ms">().set(now_ms); + pong.get<"live">() = make_live(now_ms); + + auto eo_out = encode_message<sch::MsgPong>(pong); + if(eo_out.is_error()){ + return {}; + } + + ++pongs_; + return eo_out.get_value(); + } + + std::string handle_welcome(uint64_t now_ms){ + // Registration accepted. Announce readiness, which is a claim only the + // worker can make -- the orchestrator will not assume it. + auto ready = make_message<sch::MsgReady>(msg::ready, cfg_.worker_id, 0u); + ready.get<"live">() = make_live(now_ms); + + auto eo = encode_message<sch::MsgReady>(ready); + if(eo.is_error()){ + return {}; + } + return eo.get_value(); + } + + config cfg_; + data<sch::WebGl2Capability> capability_; + bool probe_ok_ = false; + uint64_t started_at_ms_ = 0u; + uint64_t pongs_ = 0u; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/worker/c++/worker.cpp b/poc/web_swarm/worker/c++/worker.cpp new file mode 100644 index 0000000..db6d2cd --- /dev/null +++ b/poc/web_swarm/worker/c++/worker.cpp @@ -0,0 +1,102 @@ +#include "protocol.hpp" + +#include <emscripten.h> + +#include <string> + +/** + * The browser worker, wasm half: a thin shim over worker_protocol. + * + * The division of labour with worker.js is deliberate and worth stating, + * because the obvious alternative is tempting and wrong. JS owns the + * transport: the WebSocket, the RTCPeerConnection, the data channel and the + * timer. It moves strings and nothing else. Everything that decides what a + * string *means* is on this side. + * + * The reason is single sourcing. The wire format is defined once, as schemas + * in modules/swarm, and both the orchestrator and this worker encode and + * decode through the same codec. A pure JS worker would have been quicker to + * write and would have put a second, hand maintained copy of the format in a + * language where nothing checks it against the first. + * + * Every entry point takes and returns plain C strings. Returned pointers point + * into a static buffer owned here and stay valid until the next call that + * returns a string, which is all JS needs since it copies immediately via + * UTF8ToString. + */ +namespace { + +kel::lbm::swarm::worker_protocol g_worker; + +/// Backing store for whatever was last handed to JS. +std::string g_out; + +const char *hand_over(std::string text){ + g_out = std::move(text); + return g_out.c_str(); +} + +/** + * Monotonic milliseconds since the page started. Deliberately performance.now + * rather than wall clock: the live block reports an uptime, and round trips + * are measured by echoing the orchestrator's own stamp, so nothing here needs + * to agree with another machine's idea of the time. + */ +EM_JS(double, js_now_ms, (), { + return performance.now(); +}); + +uint64_t now_ms(){ + return static_cast<uint64_t>(js_now_ms()); +} + +} // namespace + +extern "C" { + +/// Runs the capability probe. Call once, before connecting. 1 if this device +/// can actually run a solver, 0 otherwise. +EMSCRIPTEN_KEEPALIVE +int kel_worker_init(const char *worker_id, const char *user_agent){ + kel::lbm::swarm::worker_protocol::config cfg; + if(worker_id != nullptr and *worker_id != '\0'){ + cfg.worker_id = worker_id; + } + if(user_agent != nullptr){ + cfg.user_agent = user_agent; + } + + return g_worker.init(std::move(cfg), now_ms()) ? 1 : 0; +} + +EMSCRIPTEN_KEEPALIVE +const char *kel_worker_hello(){ + return hand_over(g_worker.hello(now_ms())); +} + +EMSCRIPTEN_KEEPALIVE +const char *kel_worker_handle(const char *text){ + if(text == nullptr){ + return hand_over(std::string{}); + } + return hand_over(g_worker.handle(std::string{text}, now_ms())); +} + +EMSCRIPTEN_KEEPALIVE +double kel_worker_pongs(){ + return static_cast<double>(g_worker.pongs()); +} + +EMSCRIPTEN_KEEPALIVE +double kel_worker_budget_bytes(){ + return static_cast<double>( + g_worker.capability().get<"budget">().get<"probed_bytes_safe">().get()); +} + +} // extern "C" + +int main(){ + // Nothing to do. The page drives everything through the exports above; + // main exists because emscripten wants an entry point. + return 0; +} diff --git a/poc/web_swarm/worker/selftest/SConscript b/poc/web_swarm/worker/selftest/SConscript new file mode 100644 index 0000000..f60d79a --- /dev/null +++ b/poc/web_swarm/worker/selftest/SConscript @@ -0,0 +1,35 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Built for node rather than the web, so a check phase can run it. Same +# protocol code as the real worker; only the entry point and the environment +# differ. +test_env = env.Clone(); +test_env['LINKFLAGS'] = [ + '-sWASM=1', + '-sENVIRONMENT=node', + '-sALLOW_MEMORY_GROWTH=1', + # The probe is compiled in and must link, but with no context it returns + # cleanly rather than running any of it. + '-sMIN_WEBGL_VERSION=2', + '-sMAX_WEBGL_VERSION=2', + '-sFULL_ES3=1', +] + +test_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); + +env.sources += test_env.sources; + +objects = [] +test_env.add_source_files(objects, test_env.sources, shared=False); +env.selftest = test_env.em_program('#bin/kel_lbm_worker_selftest', [objects]); + +env.targets += [env.selftest]; diff --git a/poc/web_swarm/worker/selftest/selftest.cpp b/poc/web_swarm/worker/selftest/selftest.cpp new file mode 100644 index 0000000..ee13bbf --- /dev/null +++ b/poc/web_swarm/worker/selftest/selftest.cpp @@ -0,0 +1,146 @@ +#include "../c++/protocol.hpp" + +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <cstdio> +#include <string> + +/** + * Node-hosted self test for the browser worker's protocol half. + * + * A check phase has no browser, so the real worker cannot be tested there. + * But only the GL probe actually needs one -- the part that decides what to + * say, and says it in the format the orchestrator parses, is ordinary code. + * Built for node and run in the derivation's check phase, this covers that + * part and leaves only the probe genuinely untested. + * + * Running with no WebGL context is not a limitation here, it is the point: it + * is the same path a browser takes when the device cannot give one, and the + * worker has to register anyway so the orchestrator learns why. + */ +namespace { + +using namespace kel::lbm; + +int failures = 0; + +void check(bool ok, const char *what){ + if(ok){ + std::printf(" ok %s\n", what); + return; + } + std::printf(" FAIL %s\n", what); + ++failures; +} + +} // namespace + +int main(){ + std::printf("worker protocol self test\n"); + + swarm::worker_protocol worker; + + swarm::worker_protocol::config cfg; + cfg.worker_id = "selftest"; + cfg.user_agent = "node"; + + // No WebGL under node, so the probe must fail -- cleanly, without taking + // the rest of the worker down with it. + const bool usable = worker.init(cfg, 1000u); + check(not usable, "probe reports the device unusable without a context"); + check(not worker.probe_ok(), "and remembers that it did"); + + // Registration still happens. An orchestrator that knows why a device is + // unusable is better off than one that never heard from it. + const std::string hello = worker.hello(2000u); + check(not hello.empty(), "an unusable device still produces a hello"); + + { + auto eo = swarm::decode_message<sch::MsgHelloWebGl2>(hello); + check(not eo.is_error(), "the hello decodes as a webgl2 hello"); + if(not eo.is_error()){ + const auto &cap = eo.get_value().get<"capability">(); + check(cap.get<"kind">().stl_view() == wkind::webgl2, + "it registers as a webgl2 worker"); + check(cap.get<"worker_id">().stl_view() == "selftest", + "it carries the configured id"); + check(cap.get<"schema_version">().stl_view() == schema_version, + "it states the schema version the orchestrator checks"); + check(not cap.get<"webgl2">().get<"features">() + .get<"color_buffer_float">().get(), + "and honestly reports no float render target"); + } + } + + // The routing decode the orchestrator does first has to work on it too. + { + auto eo = swarm::decode_message<sch::MsgHello>(hello); + check(not eo.is_error(), "the orchestrator's routing decode accepts it"); + } + + // A ping must come back as a pong echoing the sequence and the timestamp. + { + auto ping = swarm::make_message<sch::MsgPing>(msg::ping, "orch", 42u); + ping.get<"sent_at_ms">().set(123456u); + + auto eo_ping = swarm::encode_message<sch::MsgPing>(ping); + check(not eo_ping.is_error(), "a ping encodes"); + + const std::string reply = worker.handle(eo_ping.get_value(), 3000u); + check(not reply.empty(), "a ping is answered"); + + auto eo_pong = swarm::decode_message<sch::MsgPong>(reply); + check(not eo_pong.is_error(), "the answer is a well formed pong"); + if(not eo_pong.is_error()){ + const auto &pong = eo_pong.get_value(); + check(pong.get<"type">().stl_view() == msg::pong, "typed as a pong"); + check(pong.get<"seq">().get() == 42u, + "echoing the sequence, so a late pong can be told apart"); + check(pong.get<"sent_at_ms">().get() == 123456u, + "echoing the timestamp, so rtt needs no clock agreement"); + check(pong.get<"worker_now_ms">().get() == 3000u, + "and carrying the worker's own clock"); + check(pong.get<"live">().get<"state">().stl_view() == wstate::error, + "reporting error state, since the probe failed"); + } + + check(worker.pongs() == 1u, "the pong is counted"); + } + + // A welcome must produce a ready. Readiness is a claim only the worker + // makes; the orchestrator never assumes it. + { + auto welcome = swarm::make_message<sch::MsgWelcome>( + msg::welcome, "w1", 0u); + welcome.get<"heartbeat_ms">().set(2000u); + welcome.get<"schema_version">().set(std::string{schema_version}); + + auto eo_wel = swarm::encode_message<sch::MsgWelcome>(welcome); + check(not eo_wel.is_error(), "a welcome encodes"); + + const std::string reply = worker.handle(eo_wel.get_value(), 4000u); + check(not reply.empty(), "a welcome is answered"); + + auto eo_ready = swarm::decode_message<sch::MsgReady>(reply); + check(not eo_ready.is_error(), "the answer is a well formed ready"); + if(not eo_ready.is_error()){ + check(eo_ready.get_value().get<"type">().stl_view() == msg::ready, + "typed as ready"); + } + } + + // Things the worker has no business answering must produce nothing rather + // than an empty message on the wire. + { + check(worker.handle("", 5000u).empty(), "garbage is not answered"); + check(worker.handle("{", 5000u).empty(), "malformed json is not answered"); + check(worker.handle(R"({"type":"pong","worker_id":"x","seq":1})", 5000u) + .empty(), + "a message meant for the orchestrator is not answered"); + check(worker.pongs() == 1u, "and none of that counted as a pong"); + } + + std::printf("%s\n", failures == 0 ? "PASS" : "FAIL"); + return failures == 0 ? 0 : 1; +} diff --git a/poc/web_swarm/worker/web/index.html b/poc/web_swarm/worker/web/index.html new file mode 100644 index 0000000..dddf4a1 --- /dev/null +++ b/poc/web_swarm/worker/web/index.html @@ -0,0 +1,87 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>kel-lbm swarm worker</title> +<style> + body { + margin: 0; + padding: 1.5rem; + background: #14161a; + color: #d8dee9; + font-family: system-ui, sans-serif; + line-height: 1.5; + } + h1 { font-size: 1.1rem; margin: 0 0 1rem; font-weight: 600; } + .row { display: flex; gap: 0.5rem; margin-bottom: 0.5rem; flex-wrap: wrap; } + label { min-width: 7rem; align-self: center; font-size: 0.85rem; color: #8c95a3; } + input { + flex: 1; min-width: 14rem; padding: 0.4rem 0.6rem; + background: #1d2027; color: #d8dee9; + border: 1px solid #333842; border-radius: 4px; + font-family: ui-monospace, monospace; font-size: 0.85rem; + } + button { + padding: 0.4rem 1rem; border-radius: 4px; cursor: pointer; + background: #2d4f74; color: #e8eef5; + border: 1px solid #3b6694; font-size: 0.85rem; + } + button:disabled { opacity: 0.4; cursor: default; } + #status { + margin: 1rem 0; padding: 0.75rem 1rem; + background: #1d2027; border-left: 3px solid #3b6694; border-radius: 3px; + } + #state { font-size: 1.05rem; font-weight: 600; } + #detail { color: #8c95a3; font-size: 0.85rem; font-family: ui-monospace, monospace; } + #log { + height: 16rem; overflow-y: auto; white-space: pre-wrap; + background: #0f1115; border: 1px solid #262b33; border-radius: 4px; + padding: 0.75rem; font-family: ui-monospace, monospace; font-size: 0.78rem; + color: #9aa5b1; + } + /* + * The probe needs a real canvas to get a WebGL2 context from, but nothing is + * ever drawn for display. Kept at one pixel and out of the way rather than + * display:none, which some browsers treat as a reason to skip the context. + */ + #kel-probe { position: absolute; width: 1px; height: 1px; opacity: 0; pointer-events: none; } + .note { color: #6b7684; font-size: 0.78rem; margin-top: 1rem; } +</style> +</head> +<body> + +<h1>kel-lbm swarm worker</h1> + +<div class="row"> + <label for="url">orchestrator</label> + <input id="url" type="text" spellcheck="false"> +</div> +<div class="row"> + <label for="worker-id">worker id</label> + <input id="worker-id" type="text" spellcheck="false"> +</div> +<div class="row"> + <button id="connect" disabled>connect</button> + <button id="disconnect" disabled>disconnect</button> +</div> + +<div id="status"> + <div id="state">loading</div> + <div id="detail"></div> +</div> + +<div id="log"></div> + +<p class="note"> + Open this page from an HTTP server, not a file:// URL — wasm is fetched, + and file:// is a null origin. The orchestrator is a separate process on its own + port; append <code>?orchestrator=ws://host:port/</code> to override. +</p> + +<canvas id="kel-probe" width="1" height="1"></canvas> + +<script src="worker.js"></script> +<script src="kel_lbm_worker.js"></script> +</body> +</html> diff --git a/poc/web_swarm/worker/web/worker.js b/poc/web_swarm/worker/web/worker.js new file mode 100644 index 0000000..04d4e10 --- /dev/null +++ b/poc/web_swarm/worker/web/worker.js @@ -0,0 +1,286 @@ +// The browser worker, transport half. +// +// This file owns the WebSocket, the RTCPeerConnection, the data channel and +// nothing else. Every decision about what a message means lives in the wasm +// module, so the wire format has exactly one definition -- the schemas in +// modules/swarm -- rather than one in C++ and a hand maintained copy here. +// +// The rule to keep: if you find yourself about to read a field out of a parsed +// message in this file, it belongs on the other side of the boundary. + +'use strict'; + +const ui = { + state: null, + detail: null, + log: null, +}; + +function setState(text, detail) { + if (ui.state) ui.state.textContent = text; + if (ui.detail && detail !== undefined) ui.detail.textContent = detail; +} + +function log(line) { + console.log(line); + if (!ui.log) return; + const stamp = new Date().toLocaleTimeString(); + ui.log.textContent += `[${stamp}] ${line}\n`; + ui.log.scrollTop = ui.log.scrollHeight; +} + +// Normalises an ICE candidate into the shape modules/swarm declares. The +// browser hands out {candidate, sdpMid, sdpMLineIndex}; the wire carries +// {type, candidate, mid}. Doing the mapping here keeps every message on the +// wire uniform -- all of them are objects with a "type". +function candidateMessage(candidate) { + return JSON.stringify({ + type: 'candidate', + candidate: candidate.candidate, + mid: candidate.sdpMid === null ? '' : candidate.sdpMid, + }); +} + +function descriptionMessage(description) { + return JSON.stringify({ + type: description.type, + sdp: description.sdp, + }); +} + +class Worker { + constructor(mod, url, workerId) { + this.mod = mod; + this.url = url; + this.workerId = workerId; + this.ws = null; + this.pc = null; + this.control = null; + + // Bound once so the wasm exports are called by name rather than through + // ccall at every message, which would re-resolve them each time. + this.hello = mod.cwrap('kel_worker_hello', 'string', []); + this.handle = mod.cwrap('kel_worker_handle', 'string', ['string']); + this.pongs = mod.cwrap('kel_worker_pongs', 'number', []); + } + + connect() { + setState('connecting', this.url); + log(`connecting to ${this.url}`); + + this.ws = new WebSocket(this.url); + + this.ws.onopen = () => { + setState('signalling'); + log('signalling socket open'); + }; + + this.ws.onerror = () => { + setState('error', 'signalling socket failed'); + log('signalling socket error'); + }; + + this.ws.onclose = () => { + setState('disconnected'); + log('signalling socket closed'); + }; + + this.ws.onmessage = (event) => this.onSignal(event.data); + + this.startPeerConnection(); + } + + startPeerConnection() { + // No ICE servers: host candidates are enough on localhost or a LAN, and + // a STUN server is only needed to get through NAT. + this.pc = new RTCPeerConnection({ iceServers: [] }); + + this.pc.onicecandidate = (event) => { + if (!event.candidate) return; + this.send(candidateMessage(event.candidate)); + }; + + this.pc.onconnectionstatechange = () => { + log(`peer connection ${this.pc.connectionState}`); + if (this.pc.connectionState === 'failed') { + setState('error', 'peer connection failed'); + } + }; + + // The orchestrator creates the channel, so it arrives rather than being + // made here. That fixes the ordering: it is already in the offer, so + // there is no race between it opening and the first send. + this.pc.ondatachannel = (event) => { + if (event.channel.label !== 'control') return; + this.attachControl(event.channel); + }; + } + + attachControl(channel) { + this.control = channel; + + channel.onopen = () => { + setState('registering'); + log('control channel open'); + + const hello = this.hello(); + if (!hello) { + setState('error', 'could not build hello'); + return; + } + log(`-> hello (${hello.length} bytes)`); + channel.send(hello); + }; + + channel.onmessage = (event) => { + if (typeof event.data !== 'string') return; + + // The one place the transport looks inside a message, and only to say + // what happened in the log. The reply itself is decided by wasm. + const reply = this.handle(event.data); + + let kind = '?'; + try { + kind = JSON.parse(event.data).type; + } catch (err) { + kind = 'unparseable'; + } + + if (kind === 'welcome') { + setState('registered', `pongs 0`); + log('<- welcome, registered'); + } else if (kind === 'ping') { + setState('registered', `pongs ${this.pongs()}`); + } else { + log(`<- ${kind}`); + } + + if (reply && channel.readyState === 'open') { + channel.send(reply); + } + }; + + channel.onclose = () => { + setState('disconnected', 'control channel closed'); + log('control channel closed'); + }; + } + + onSignal(text) { + let message; + try { + message = JSON.parse(text); + } catch (err) { + log('malformed signalling message'); + return; + } + + if (message.type === 'offer' || message.type === 'answer') { + this.pc + .setRemoteDescription({ type: message.type, sdp: message.sdp }) + .then(() => { + if (message.type !== 'offer') return null; + return this.pc.createAnswer(); + }) + .then((answer) => { + if (!answer) return null; + return this.pc.setLocalDescription(answer).then(() => { + this.send(descriptionMessage(this.pc.localDescription)); + }); + }) + .catch((err) => log(`signalling failed: ${err}`)); + return; + } + + if (message.type === 'candidate') { + this.pc + .addIceCandidate({ + candidate: message.candidate, + sdpMid: message.mid === '' ? null : message.mid, + }) + .catch((err) => log(`bad candidate: ${err}`)); + return; + } + + log(`unknown signalling type: ${message.type}`); + } + + send(text) { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(text); + } + } + + disconnect() { + if (this.control) this.control.close(); + if (this.pc) this.pc.close(); + if (this.ws) this.ws.close(); + setState('disconnected'); + } +} + +function defaultUrl() { + const params = new URLSearchParams(window.location.search); + const fromQuery = params.get('orchestrator'); + if (fromQuery) return fromQuery; + // The page is served by something else -- a WebSocket server is not an HTTP + // server -- so the orchestrator is not on this port. 8080 is its default. + return `ws://${window.location.hostname || '127.0.0.1'}:8080/`; +} + +window.addEventListener('DOMContentLoaded', () => { + ui.state = document.getElementById('state'); + ui.detail = document.getElementById('detail'); + ui.log = document.getElementById('log'); + + const urlInput = document.getElementById('url'); + const idInput = document.getElementById('worker-id'); + const connectButton = document.getElementById('connect'); + const disconnectButton = document.getElementById('disconnect'); + + urlInput.value = defaultUrl(); + idInput.value = `browser-${Math.floor(Math.random() * 10000)}`; + + setState('loading', 'instantiating wasm'); + + Module().then((mod) => { + const init = mod.cwrap('kel_worker_init', 'number', ['string', 'string']); + const budget = mod.cwrap('kel_worker_budget_bytes', 'number', []); + + log('wasm ready, probing device'); + + const usable = init(idInput.value, navigator.userAgent); + const mib = Math.round(budget() / (1024 * 1024)); + + if (usable) { + setState('probed', `${mib} MiB usable`); + log(`probe: usable, ${mib} MiB safe budget`); + } else { + // Still worth connecting. An orchestrator that knows why a device is + // unusable is better off than one that never heard from it. + setState('probed', 'device cannot run a solver'); + log('probe: device unusable (no float render target, or no context)'); + } + + let worker = null; + + connectButton.disabled = false; + connectButton.onclick = () => { + if (worker) worker.disconnect(); + worker = new Worker(mod, urlInput.value, idInput.value); + worker.connect(); + connectButton.disabled = true; + disconnectButton.disabled = false; + }; + + disconnectButton.onclick = () => { + if (worker) worker.disconnect(); + worker = null; + connectButton.disabled = false; + disconnectButton.disabled = true; + }; + }).catch((err) => { + setState('error', 'wasm failed to load'); + log(`wasm load failed: ${err}`); + }); +}); diff --git a/poc/web_swarm/worker_native/c++/SConscript b/poc/web_swarm/worker_native/c++/SConscript new file mode 100644 index 0000000..873e075 --- /dev/null +++ b/poc/web_swarm/worker_native/c++/SConscript @@ -0,0 +1,26 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +worker_env = env.Clone(); + +worker_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +worker_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += worker_env.sources; +env.headers += worker_env.headers; + +objects = [] +worker_env.add_source_files(objects, worker_env.sources, shared=False); +env.worker_native = worker_env.Program('#bin/kel_lbm_worker_native', [objects]); + +env.targets += [env.worker_native]; + +env.Install('$prefix/bin/', env.worker_native); diff --git a/poc/web_swarm/worker_native/c++/capability.hpp b/poc/web_swarm/worker_native/c++/capability.hpp new file mode 100644 index 0000000..d892a8f --- /dev/null +++ b/poc/web_swarm/worker_native/c++/capability.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include <kel/lbm/swarm/capability.hpp> +#include <kel/lbm/abstract/data.hpp> + +#include <string> +#include <thread> + +#if defined(__linux__) +#include <unistd.h> +#endif + +namespace kel { +namespace lbm { +namespace swarm { + +/** + * What a native worker can honestly say about itself. + * + * Deliberately short. A browser worker has to measure its texture budget + * because WebGL will not tell it, but a process can simply ask the OS, and + * there is no point inventing numbers beyond that until there is a scheduler + * that would use them. + */ +inline data<sch::NativeCapability> probe_native_capability(){ + data<sch::NativeCapability> cap; + + unsigned threads = std::thread::hardware_concurrency(); + if(threads == 0u){ + // hardware_concurrency is allowed to give up; one thread is the + // honest floor rather than zero, which would read as "no capacity". + threads = 1u; + } + cap.get<"threads">().set(threads); + + uint64_t memory_bytes = 0u; +#if defined(__linux__) + const long pages = ::sysconf(_SC_PHYS_PAGES); + const long page_size = ::sysconf(_SC_PAGE_SIZE); + if(pages > 0 and page_size > 0){ + memory_bytes = static_cast<uint64_t>(pages) * static_cast<uint64_t>(page_size); + } +#endif + cap.get<"memory_bytes">().set(memory_bytes); + +#if defined(__x86_64__) + cap.get<"cpu">().set(std::string{"x86_64"}); +#elif defined(__aarch64__) + cap.get<"cpu">().set(std::string{"aarch64"}); +#else + cap.get<"cpu">().set(std::string{"unknown"}); +#endif + + // Nothing here links SYCL yet. Reported so the orchestrator can start + // distinguishing compute capable workers without a protocol change. + cap.get<"has_sycl">().set(false); + + return cap; +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/worker_native/c++/main.cpp b/poc/web_swarm/worker_native/c++/main.cpp new file mode 100644 index 0000000..6f4fbc8 --- /dev/null +++ b/poc/web_swarm/worker_native/c++/main.cpp @@ -0,0 +1,403 @@ +#include "capability.hpp" + +#include <rtc/rtc.hpp> + +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/signal.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <cstdlib> +#include <iostream> +#include <memory> +#include <mutex> +#include <string> + +/** + * A native worker. + * + * Two jobs, and it is worth being explicit that the second is not a lesser one: + * + * 1. It is a worker kind. A process can join the swarm and be handed work, + * and it registers as kind "native" with the capability a process can + * actually report -- threads and memory -- rather than impersonating a + * browser and claiming a texture budget it has no way to measure. + * + * 2. It is the only end to end coverage the orchestrator can have in CI. The + * browser worker needs a browser, which a nix check phase does not have, + * so this is what keeps the signalling, the peer connection and the + * control channel under test. + * + * The transport is the mirror image of the orchestrator's: connect the + * signalling socket, answer the offer, wait for the channel the orchestrator + * created, then speak the control protocol on it. + */ +namespace { + +using namespace kel::lbm; + +struct worker_config { + std::string url = "ws://127.0.0.1:8080/"; + std::string worker_id = "native"; + uint64_t timeout_ms = 15000u; + + /** + * Exit once the hello has been answered. This is what the check script + * runs; left off, the worker stays connected like a real one. + */ + bool once = false; + + /** + * Exit after answering this many pings. Zero means never. Takes + * precedence over --once, so a check can cover registration and liveness + * in a single run rather than two. + */ + uint64_t pongs_target = 0u; +}; + +std::atomic<uint64_t> g_pongs_sent{0u}; + +std::mutex g_mutex; +std::condition_variable g_cv; +std::atomic<bool> g_done{false}; +std::atomic<int> g_exit_code{1}; + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +void finish(int code){ + g_exit_code.store(code); + g_done.store(true); + g_cv.notify_all(); +} + +std::string text_of(const rtc::message_variant &msg){ + if(std::holds_alternative<rtc::string>(msg)){ + return std::get<rtc::string>(msg); + } + return {}; +} + +data<sch::WorkerLive> make_live(uint64_t started_at){ + data<sch::WorkerLive> live; + live.get<"state">().set(std::string{wstate::ready}); + live.get<"uptime_ms">().set(now_ms() - started_at); + live.get<"steps_done">().set(0u); + live.get<"last_error">().set(std::string{""}); + return live; +} + +void print_usage(const char *argv0){ + std::cout + << "Usage: " << argv0 << " [options]\n" + << "\n" + << " --url <ws url> Orchestrator signalling socket\n" + << " (default ws://127.0.0.1:8080/).\n" + << " --id <name> Worker id to report (default 'native').\n" + << " --timeout <ms> Give up if registration has not completed\n" + << " (default 15000).\n" + << " --once Exit 0 once the hello has been answered.\n" + << " Used by the check; without it the worker\n" + << " stays connected.\n" + << " --pongs <n> Exit 0 after answering n pings. Takes\n" + << " precedence over --once, so one run can cover\n" + << " registration and liveness together.\n" + << " --help This text.\n"; +} + +} // namespace + +int main(int argc, char **argv){ + worker_config cfg; + + for(int i = 1; i < argc; ++i){ + const std::string arg{argv[i]}; + + if(arg == "--help" or arg == "-h"){ + print_usage(argv[0]); + return 0; + } + if(arg == "--url" and (i + 1) < argc){ + cfg.url = argv[++i]; + continue; + } + if(arg == "--id" and (i + 1) < argc){ + cfg.worker_id = argv[++i]; + continue; + } + if(arg == "--timeout" and (i + 1) < argc){ + cfg.timeout_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--once"){ + cfg.once = true; + continue; + } + if(arg == "--pongs" and (i + 1) < argc){ + cfg.pongs_target = std::strtoull(argv[++i], nullptr, 10); + continue; + } + + std::cerr << "Unknown or incomplete option: " << arg << "\n\n"; + print_usage(argv[0]); + return 2; + } + + const uint64_t started_at = now_ms(); + + auto ws = std::make_shared<rtc::WebSocket>(); + auto pc = std::make_shared<rtc::PeerConnection>(); + + /** + * The orchestrator creates the control channel, so the worker waits for it + * rather than making one. Held here so the callbacks below outlive the + * negotiation. + */ + std::shared_ptr<rtc::DataChannel> control; + std::mutex control_mutex; + + pc->onLocalDescription([ws](rtc::Description desc){ + if(not ws->isOpen()){ + return; + } + data<sch::SignalDescription> out; + out.get<"type">().set(desc.typeString()); + out.get<"sdp">().set(std::string{desc}); + + auto eo = swarm::encode_message<sch::SignalDescription>(out); + if(eo.is_error()){ + return; + } + ws->send(eo.get_value()); + }); + + pc->onLocalCandidate([ws](rtc::Candidate cand){ + if(not ws->isOpen()){ + return; + } + data<sch::SignalCandidate> out; + out.get<"type">().set(std::string{sig::candidate}); + out.get<"candidate">().set(cand.candidate()); + out.get<"mid">().set(cand.mid()); + + auto eo = swarm::encode_message<sch::SignalCandidate>(out); + if(eo.is_error()){ + return; + } + ws->send(eo.get_value()); + }); + + pc->onStateChange([](rtc::PeerConnection::State state){ + if(state == rtc::PeerConnection::State::Failed){ + std::cerr << "peer connection failed\n"; + finish(1); + } + }); + + pc->onDataChannel([&, started_at](std::shared_ptr<rtc::DataChannel> dc){ + if(dc->label() != "control"){ + // Nothing else is expected; ignoring rather than failing keeps the + // worker forward compatible with an orchestrator that adds one. + return; + } + + { + std::lock_guard<std::mutex> lock{control_mutex}; + control = dc; + } + + dc->onOpen([dc, &cfg, started_at](){ + std::cout << "control channel open" << std::endl; + + auto hello = swarm::make_message<sch::MsgHelloNative>( + msg::hello, cfg.worker_id, 0u); + + auto &cap = hello.get<"capability">(); + cap.get<"worker_id">().set(cfg.worker_id); + cap.get<"kind">().set(std::string{wkind::native}); + cap.get<"schema_version">().set(std::string{schema_version}); + cap.get<"reported_at_ms">().set(now_ms()); + cap.get<"user_agent">().set(std::string{"kel-lbm-worker-native"}); + cap.get<"native">() = swarm::probe_native_capability(); + + auto eo = swarm::encode_message<sch::MsgHelloNative>(hello); + if(eo.is_error()){ + std::cerr << "could not encode hello\n"; + finish(1); + return; + } + + std::cout << "-> hello (" << eo.get_value().size() << " bytes)" + << std::endl; + dc->send(eo.get_value()); + }); + + dc->onMessage([dc, &cfg, started_at](rtc::message_variant raw){ + const std::string text = text_of(raw); + if(text.empty()){ + return; + } + + auto eo_hdr = swarm::peek_header(text); + if(eo_hdr.is_error()){ + std::cerr << "malformed control message\n"; + finish(1); + return; + } + + const std::string type = + eo_hdr.get_value().get<"type">().stl_string(); + std::cout << "<- " << type << std::endl; + + /** + * Step 2's orchestrator echoes rather than replying, so a hello + * coming back is what confirms the round trip. Once the registry + * lands this becomes a welcome, and both are accepted so the two + * can be developed in either order. + */ + if(type == msg::hello or type == msg::welcome){ + // Registration is done. If the run is also meant to cover + // liveness, stay for the pings instead of leaving now. + if(cfg.once and cfg.pongs_target == 0u){ + finish(0); + } + return; + } + + if(type == msg::ping){ + auto eo_ping = swarm::decode_message<sch::MsgPing>(text); + if(eo_ping.is_error()){ + std::cerr << "bad ping\n"; + return; + } + + auto pong = swarm::make_message<sch::MsgPong>( + msg::pong, cfg.worker_id, + eo_ping.get_value().get<"seq">().get()); + pong.get<"sent_at_ms">().set( + eo_ping.get_value().get<"sent_at_ms">().get()); + pong.get<"worker_now_ms">().set(now_ms()); + pong.get<"live">() = make_live(started_at); + + auto eo_out = swarm::encode_message<sch::MsgPong>(pong); + if(eo_out.is_error()){ + return; + } + + // Reply on the channel the message arrived on rather than + // looking it up: same channel, no lock, no lifetime question. + if(dc->isOpen()){ + dc->send(eo_out.get_value()); + const uint64_t sent = g_pongs_sent.fetch_add(1u) + 1u; + if(cfg.pongs_target > 0u and sent >= cfg.pongs_target){ + std::cout << "answered " << sent << " ping(s)" + << std::endl; + finish(0); + } + } + return; + } + }); + + dc->onClosed([](){ + std::cout << "control channel closed" << std::endl; + finish(g_exit_code.load()); + }); + }); + + ws->onMessage([pc](rtc::message_variant raw){ + const std::string text = text_of(raw); + if(text.empty()){ + return; + } + + auto eo_hdr = swarm::decode_message<sch::SignalHeader>(text); + if(eo_hdr.is_error()){ + std::cerr << "malformed signalling message\n"; + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == sig::offer or type == sig::answer){ + auto eo = swarm::decode_message<sch::SignalDescription>(text); + if(eo.is_error()){ + std::cerr << "bad " << type << "\n"; + return; + } + try { + // Auto negotiation turns setting a remote offer into an answer. + pc->setRemoteDescription( + rtc::Description{eo.get_value().get<"sdp">().stl_string(), type}); + } catch (const std::exception &e){ + std::cerr << "setRemoteDescription: " << e.what() << "\n"; + } + return; + } + + if(type == sig::candidate){ + auto eo = swarm::decode_message<sch::SignalCandidate>(text); + if(eo.is_error()){ + std::cerr << "bad candidate\n"; + return; + } + try { + pc->addRemoteCandidate( + rtc::Candidate{eo.get_value().get<"candidate">().stl_string(), + eo.get_value().get<"mid">().stl_string()}); + } catch (const std::exception &e){ + std::cerr << "addRemoteCandidate: " << e.what() << "\n"; + } + return; + } + }); + + ws->onClosed([](){ + // Closing before the exchange completed is a failure; after it, the + // exit code is already set and this is just teardown. + finish(g_exit_code.load()); + }); + + ws->onError([](std::string err){ + std::cerr << "signalling error: " << err << "\n"; + finish(1); + }); + + std::cout << "connecting to " << cfg.url << std::endl; + try { + ws->open(cfg.url); + } catch (const std::exception &e){ + std::cerr << "[Error] Could not open " << cfg.url << ": " << e.what() << "\n"; + return 1; + } + + { + std::unique_lock<std::mutex> lock{g_mutex}; + const bool finished = g_cv.wait_for( + lock, std::chrono::milliseconds{cfg.timeout_ms}, + [](){ return g_done.load(); }); + + if(not finished){ + std::cerr << "[Error] Timed out after " << cfg.timeout_ms + << "ms without completing registration\n"; + g_exit_code.store(1); + } + } + + // Tear down before the callbacks' captures go out of scope. + { + std::lock_guard<std::mutex> lock{control_mutex}; + if(control){ + control->close(); + } + } + pc->close(); + ws->close(); + + return g_exit_code.load(); +} |
