diff options
Diffstat (limited to 'poc/web_swarm/orchestrator')
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/SConscript | 26 | ||||
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/control.cpp | 276 | ||||
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/control.hpp | 74 | ||||
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/main.cpp | 205 | ||||
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/orchestrator.cpp | 339 | ||||
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/orchestrator.hpp | 113 | ||||
| -rw-r--r-- | poc/web_swarm/orchestrator/c++/session.hpp | 49 |
7 files changed, 1082 insertions, 0 deletions
diff --git a/poc/web_swarm/orchestrator/c++/SConscript b/poc/web_swarm/orchestrator/c++/SConscript new file mode 100644 index 0000000..c4a7608 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/SConscript @@ -0,0 +1,26 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +orch_env = env.Clone(); + +orch_env.sources = sorted(glob.glob(dir_path + "/*.cpp")); +orch_env.headers = sorted(glob.glob(dir_path + "/*.hpp")); + +env.sources += orch_env.sources; +env.headers += orch_env.headers; + +objects = [] +orch_env.add_source_files(objects, orch_env.sources, shared=False); +env.orchestrator = orch_env.Program('#bin/kel_lbm_orchestrator', [objects]); + +env.targets += [env.orchestrator]; + +env.Install('$prefix/bin/', env.orchestrator); diff --git a/poc/web_swarm/orchestrator/c++/control.cpp b/poc/web_swarm/orchestrator/c++/control.cpp new file mode 100644 index 0000000..6e2d7b7 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/control.cpp @@ -0,0 +1,276 @@ +#include "control.hpp" + +#include <kel/lbm/swarm/interpret.hpp> +#include <kel/lbm/swarm/message.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <chrono> +#include <iomanip> +#include <sstream> +#include <utility> + +namespace kel { +namespace lbm { +namespace swarm { + +namespace { + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +} // namespace + +control::control(orchestrator &orch, control_config cfg) + : orch_{orch}, cfg_{std::move(cfg)}, registry_{cfg_.limits} {} + +control::~control(){ + stop_heartbeat(); +} + +void control::on_event(std::function<void(const std::string &)> handler){ + on_event_ = std::move(handler); +} + +void control::emit(const std::string &line){ + if(on_event_){ + on_event_(line); + } +} + +void control::attach(){ + orch_.on_channel_open([this](const std::shared_ptr<session> &sess){ + registry_.add_session(sess->id, now_ms()); + emit("[" + sess->id + "] channel open, awaiting hello"); + }); + + orch_.on_channel_closed([this](const std::shared_ptr<session> &sess){ + auto rec = registry_.remove(sess->id); + if(rec){ + emit("[" + sess->id + "] disconnected (" + + (rec->worker_id.empty() ? "unregistered" : rec->worker_id) + ")"); + } + }); + + orch_.on_message([this](const std::shared_ptr<session> &sess, + const std::string &text){ + handle_message(sess, text); + }); +} + +void control::handle_message(const std::shared_ptr<session> &sess, + const std::string &text){ + auto eo_hdr = peek_header(text); + if(eo_hdr.is_error()){ + emit("[" + sess->id + "] malformed control message, ignoring"); + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + const uint64_t seq = eo_hdr.get_value().get<"seq">().get(); + + if(type == msg::hello){ + handle_hello(sess, text); + return; + } + + if(type == msg::pong){ + auto eo = decode_message<sch::MsgPong>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed pong, ignoring"); + return; + } + registry_.record_pong(sess->id, seq, + eo.get_value().get<"sent_at_ms">().get(), + eo.get_value().get<"live">(), now_ms()); + return; + } + + if(type == msg::ready){ + auto eo = decode_message<sch::MsgReady>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed ready, ignoring"); + return; + } + registry_.mark_ready(sess->id, eo.get_value().get<"live">(), now_ms()); + emit("[" + sess->id + "] ready"); + return; + } + + if(type == msg::status){ + auto eo = decode_message<sch::MsgStatus>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] malformed status, ignoring"); + return; + } + const auto &live = eo.get_value().get<"live">(); + registry_.update_live(sess->id, live, now_ms()); + + const std::string last_error = live.get<"last_error">().stl_string(); + if(not last_error.empty()){ + emit("[" + sess->id + "] reports error: " + last_error); + } + return; + } + + emit("[" + sess->id + "] unknown message type '" + type + "', ignoring"); +} + +void control::handle_hello(const std::shared_ptr<session> &sess, + const std::string &text){ + // Routing pass: the common capability fields are enough to learn the kind, + // and the backend section is skipped because it is not named here. + auto eo_route = decode_message<sch::MsgHello>(text); + if(eo_route.is_error()){ + emit("[" + sess->id + "] hello missing its common fields, rejecting"); + return; + } + + const auto &common = eo_route.get_value().get<"capability">(); + const std::string kind = common.get<"kind">().stl_string(); + const std::string worker_id = common.get<"worker_id">().stl_string(); + const std::string their_version = common.get<"schema_version">().stl_string(); + + if(their_version != schema_version){ + // Caught deliberately rather than suffered as a decode failure later. + emit("[" + sess->id + "] schema version mismatch: worker says '" + + their_version + "', orchestrator is '" + std::string{schema_version} + + "' -- rejecting"); + return; + } + + uint64_t cells = 0u; + std::string summary; + + if(kind == wkind::webgl2){ + auto eo = decode_message<sch::MsgHelloWebGl2>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] hello claims webgl2 but has no usable " + "webgl2 section, rejecting"); + return; + } + const auto &gl = eo.get_value().get<"capability">().get<"webgl2">(); + cells = webgl2_capacity_cells(gl); + summary = webgl2_summary(gl); + + } else if(kind == wkind::native){ + auto eo = decode_message<sch::MsgHelloNative>(text); + if(eo.is_error()){ + emit("[" + sess->id + "] hello claims native but has no usable " + "native section, rejecting"); + return; + } + const auto &nat = eo.get_value().get<"capability">().get<"native">(); + cells = native_capacity_cells(nat); + summary = native_summary(nat); + + } else { + // Identifiable but unusable, which is a better outcome than a zeroed + // registration and is exactly what the routing decode is for. + emit("[" + sess->id + "] unsupported worker kind '" + kind + + "', rejecting"); + return; + } + + const uint64_t t = now_ms(); + if(not registry_.mark_registered(sess->id, worker_id, kind, cells, + summary, t)){ + return; + } + + auto welcome = make_message<sch::MsgWelcome>(msg::welcome, sess->id, 0u); + welcome.get<"heartbeat_ms">().set(cfg_.heartbeat_ms); + welcome.get<"schema_version">().set(std::string{schema_version}); + + auto eo_out = encode_message<sch::MsgWelcome>(welcome); + if(eo_out.is_error()){ + return; + } + orch_.send(sess, eo_out.get_value()); + + std::ostringstream line; + line << "[" << sess->id << "] registered " << worker_id + << " (" << kind << ") -- " << summary; + emit(line.str()); +} + +void control::start_heartbeat(){ + if(running_.exchange(true)){ + return; + } + ticker_ = std::thread{[this](){ heartbeat_loop(); }}; +} + +void control::stop_heartbeat(){ + if(not running_.exchange(false)){ + return; + } + tick_cv_.notify_all(); + if(ticker_.joinable()){ + ticker_.join(); + } +} + +void control::heartbeat_loop(){ + while(running_.load()){ + { + std::unique_lock<std::mutex> lock{tick_mutex_}; + tick_cv_.wait_for(lock, std::chrono::milliseconds{cfg_.heartbeat_ms}, + [this](){ return not running_.load(); }); + } + if(not running_.load()){ + break; + } + + const uint64_t t = now_ms(); + + // The registry decides; this loop only carries out the decisions. In + // particular nothing is sent while its lock is held. + auto result = registry_.tick(t); + + for(const auto &id : result.newly_stale){ + emit("[" + id + "] stale, no response to the last " + + std::to_string(cfg_.limits.stale_after) + " pings"); + } + + for(const auto &rec : result.dropped){ + emit("[" + rec.session_id + "] dropped after " + + std::to_string(cfg_.limits.drop_after) + " missed pings (" + + (rec.worker_id.empty() ? "unregistered" : rec.worker_id) + ")"); + + // Tear the transport down too, or the session lingers with nothing + // watching it. + if(auto sess = orch_.find_session(rec.session_id)){ + if(sess->dc){ + sess->dc->close(); + } + if(sess->pc){ + sess->pc->close(); + } + } + } + + for(const auto &target : result.to_ping){ + auto sess = orch_.find_session(target.session_id); + if(not sess){ + continue; + } + + auto ping = make_message<sch::MsgPing>(msg::ping, target.session_id, + target.seq); + ping.get<"sent_at_ms">().set(t); + + auto eo = encode_message<sch::MsgPing>(ping); + if(eo.is_error()){ + continue; + } + orch_.send(sess, eo.get_value()); + } + } +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/control.hpp b/poc/web_swarm/orchestrator/c++/control.hpp new file mode 100644 index 0000000..9365ef9 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/control.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "orchestrator.hpp" +#include <kel/lbm/swarm/registry.hpp> + +#include <atomic> +#include <condition_variable> +#include <cstdint> +#include <functional> +#include <mutex> +#include <string> +#include <thread> + +namespace kel { +namespace lbm { +namespace swarm { + +struct control_config { + uint64_t heartbeat_ms = 2000u; + registry::thresholds limits; +}; + +/** + * The control protocol on top of the transport. + * + * Kept apart from `orchestrator` so each can be reasoned about alone: the + * transport knows nothing about hellos, and this knows nothing about ICE. + * + * Owns the heartbeat thread. libdatachannel provides no timers, so liveness is + * a plain ticker -- which is fine, since the work it does per beat is a lock, a + * vector copy and some sends. + */ +class control { +public: + control(orchestrator &orch, control_config cfg); + ~control(); + + control(const control &) = delete; + control &operator=(const control &) = delete; + + /// Installs the handlers on the orchestrator. Call before listen(). + void attach(); + + void start_heartbeat(); + void stop_heartbeat(); + + registry &workers() { return registry_; } + + /// Emitted for anything worth a line in the log. + void on_event(std::function<void(const std::string &)> handler); + +private: + void handle_message(const std::shared_ptr<session> &sess, + const std::string &text); + void handle_hello(const std::shared_ptr<session> &sess, + const std::string &text); + void heartbeat_loop(); + void emit(const std::string &line); + + orchestrator &orch_; + control_config cfg_; + registry registry_; + + std::thread ticker_; + std::mutex tick_mutex_; + std::condition_variable tick_cv_; + std::atomic<bool> running_{false}; + + std::function<void(const std::string &)> on_event_; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/main.cpp b/poc/web_swarm/orchestrator/c++/main.cpp new file mode 100644 index 0000000..4400a0d --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/main.cpp @@ -0,0 +1,205 @@ +#include "control.hpp" +#include "orchestrator.hpp" + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <csignal> +#include <cstdlib> +#include <iostream> +#include <mutex> +#include <string> + +/** + * The swarm orchestrator. + * + * Accepts workers over a WebSocket, negotiates a peer connection, opens a + * control data channel, registers whatever capability the worker reports, and + * keeps the connection under a heartbeat so liveness is a fact rather than an + * assumption. + * + * There is no event loop of our own. libdatachannel runs its own threads and + * the heartbeat runs on another, so main's whole job is to start things, print + * what happens, and wait for a signal. + */ +namespace { + +std::mutex g_wait_mutex; +std::condition_variable g_wait_cv; +std::atomic<bool> g_running{true}; +std::atomic<bool> g_dump_requested{false}; + +/** + * Signal handlers may only touch lock free atomics, so these set a flag and + * poke the condition variable. The notify is outside what the standard blesses + * in a handler, but the predicate is re-checked on a timed wakeup, so a lost + * notify costs at most the poll interval rather than a hang. + */ +extern "C" void handle_stop(int){ + g_running.store(false); + g_wait_cv.notify_all(); +} + +extern "C" void handle_dump(int){ + g_dump_requested.store(true); + g_wait_cv.notify_all(); +} + +uint64_t now_ms(){ + using namespace std::chrono; + return static_cast<uint64_t>( + duration_cast<milliseconds>(steady_clock::now().time_since_epoch()).count()); +} + +void print_usage(const char *argv0){ + std::cout + << "Usage: " << argv0 << " [options]\n" + << "\n" + << " --port <n> Port for the signalling WebSocket\n" + << " (default 8080). 0 asks the OS for a free one.\n" + << " --stun <url> STUN server, e.g.\n" + << " stun:stun.l.google.com:19302. Not needed on\n" + << " localhost or a LAN.\n" + << " --heartbeat <ms> Ping interval (default 2000).\n" + << " --stale-after <n> Missed pings before a worker is called stale\n" + << " (default 3).\n" + << " --drop-after <n> Missed pings before it is dropped (default 6).\n" + << " --monitor <ms> Print the worker table this often. 0 disables\n" + << " (default 5000). Nothing is printed while no\n" + << " workers are connected.\n" + << " --help This text.\n" + << "\n" + << "SIGUSR1 prints the worker table immediately.\n" + << "\n" + << "The worker page is not served from here -- a WebSocket server is\n" + << "not an HTTP server. Serve it separately, e.g.\n" + << " python3 -m http.server 8000\n"; +} + +} // namespace + +int main(int argc, char **argv){ + using namespace kel::lbm; + + swarm::orchestrator_config ocfg; + swarm::control_config ccfg; + uint64_t monitor_ms = 5000u; + + for(int i = 1; i < argc; ++i){ + const std::string arg{argv[i]}; + + if(arg == "--help" or arg == "-h"){ + print_usage(argv[0]); + return 0; + } + if(arg == "--port" and (i + 1) < argc){ + ocfg.port = static_cast<uint16_t>(std::strtoul(argv[++i], nullptr, 10)); + continue; + } + if(arg == "--stun" and (i + 1) < argc){ + ocfg.stun_url = argv[++i]; + continue; + } + if(arg == "--heartbeat" and (i + 1) < argc){ + ccfg.heartbeat_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--stale-after" and (i + 1) < argc){ + ccfg.limits.stale_after = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--drop-after" and (i + 1) < argc){ + ccfg.limits.drop_after = std::strtoull(argv[++i], nullptr, 10); + continue; + } + if(arg == "--monitor" and (i + 1) < argc){ + monitor_ms = std::strtoull(argv[++i], nullptr, 10); + continue; + } + + std::cerr << "Unknown or incomplete option: " << arg << "\n\n"; + print_usage(argv[0]); + return 2; + } + + if(ccfg.heartbeat_ms == 0u){ + std::cerr << "--heartbeat must be greater than zero\n"; + return 2; + } + if(ccfg.limits.drop_after < ccfg.limits.stale_after){ + // Otherwise a worker is dropped before it can ever be reported stale, + // which makes the stale state unreachable and the log misleading. + std::cerr << "--drop-after must be at least --stale-after\n"; + return 2; + } + + std::signal(SIGINT, handle_stop); + std::signal(SIGTERM, handle_stop); + std::signal(SIGUSR1, handle_dump); + + swarm::orchestrator orch{ocfg}; + swarm::control ctl{orch, ccfg}; + + ctl.on_event([](const std::string &line){ + std::cout << line << std::endl; + }); + + ctl.attach(); + + uint16_t port = 0u; + try { + port = orch.listen(); + } catch (const std::exception &e){ + std::cerr << "[Error] Could not listen: " << e.what() << "\n"; + return 1; + } + + ctl.start_heartbeat(); + + std::cout << "orchestrator listening on ws://0.0.0.0:" << port << "\n" + << "heartbeat " << ccfg.heartbeat_ms << "ms, stale after " + << ccfg.limits.stale_after << " misses, drop after " + << ccfg.limits.drop_after << "\n" + << "ctrl-c to stop, SIGUSR1 for the worker table" << std::endl; + + uint64_t last_monitor = now_ms(); + + { + std::unique_lock<std::mutex> lock{g_wait_mutex}; + while(g_running.load()){ + g_wait_cv.wait_for(lock, std::chrono::milliseconds{250}, + [](){ return not g_running.load() + or g_dump_requested.load(); }); + + const bool asked = g_dump_requested.exchange(false); + const uint64_t t = now_ms(); + const bool due = (monitor_ms > 0u) and ((t - last_monitor) >= monitor_ms); + + if(not g_running.load()){ + break; + } + if(not asked and not due){ + continue; + } + + last_monitor = t; + + auto snapshot = ctl.workers().snapshot(); + // An idle orchestrator should be quiet; a periodic table of + // nothing is just log noise. An explicit SIGUSR1 always prints, + // because then someone is actually asking. + if(snapshot.empty() and not asked){ + continue; + } + + std::cout << "\n" << snapshot.size() << " worker(s):\n" + << swarm::render_table(snapshot, t) << std::endl; + } + } + + std::cout << "\nstopping" << std::endl; + ctl.stop_heartbeat(); + orch.stop(); + + return 0; +} diff --git a/poc/web_swarm/orchestrator/c++/orchestrator.cpp b/poc/web_swarm/orchestrator/c++/orchestrator.cpp new file mode 100644 index 0000000..68ed8da --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/orchestrator.cpp @@ -0,0 +1,339 @@ +#include "orchestrator.hpp" + +#include <kel/lbm/swarm/signal.hpp> +#include <kel/lbm/swarm/wire.hpp> + +#include <iostream> +#include <utility> + +namespace kel { +namespace lbm { +namespace swarm { + +namespace { + +/** + * libdatachannel delivers either text or binary. The control and signalling + * protocols are both JSON text, so binary is simply not ours and is dropped. + */ +std::string text_of(const rtc::message_variant &msg){ + if(std::holds_alternative<rtc::string>(msg)){ + return std::get<rtc::string>(msg); + } + return {}; +} + +} // namespace + +orchestrator::orchestrator(orchestrator_config cfg) : cfg_{std::move(cfg)} {} + +orchestrator::~orchestrator(){ + stop(); +} + +void orchestrator::on_message(message_handler handler){ + on_message_ = std::move(handler); +} + +void orchestrator::on_channel_open( + std::function<void(const std::shared_ptr<session> &)> handler){ + on_open_ = std::move(handler); +} + +void orchestrator::on_channel_closed( + std::function<void(const std::shared_ptr<session> &)> handler){ + on_closed_ = std::move(handler); +} + +std::string orchestrator::next_session_id(){ + // Called under the lock. + return "w" + std::to_string(++next_id_); +} + +uint16_t orchestrator::listen(){ + rtc::WebSocketServerConfiguration wscfg; + wscfg.port = cfg_.port; + wscfg.enableTls = false; + + server_ = std::make_unique<rtc::WebSocketServer>(std::move(wscfg)); + server_->onClient([this](std::shared_ptr<rtc::WebSocket> ws){ + accept_client(std::move(ws)); + }); + + return server_->port(); +} + +void orchestrator::accept_client(std::shared_ptr<rtc::WebSocket> ws){ + auto sess = std::make_shared<session>(); + sess->ws = std::move(ws); + + { + std::lock_guard<std::mutex> lock{mutex_}; + sess->id = next_session_id(); + sessions_.emplace(sess->id, sess); + } + + // Callbacks capture weakly. A worker that drops mid-negotiation would + // otherwise keep its own session alive through the capture and never be + // collected. + std::weak_ptr<session> weak = sess; + + sess->ws->onMessage([this, weak](rtc::message_variant msg){ + auto s = weak.lock(); + if(not s){ + return; + } + const std::string text = text_of(msg); + if(text.empty()){ + return; + } + handle_signal(s, text); + }); + + sess->ws->onClosed([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + // The data channel may already be gone; dropping the session is + // idempotent so the two paths cannot double-remove. + drop_session(s->id); + }); + + begin_negotiation(sess); +} + +void orchestrator::begin_negotiation(const std::shared_ptr<session> &sess){ + rtc::Configuration rtccfg; + if(not cfg_.stun_url.empty()){ + rtccfg.iceServers.emplace_back(cfg_.stun_url); + } + + sess->pc = std::make_shared<rtc::PeerConnection>(rtccfg); + + std::weak_ptr<session> weak = sess; + + sess->pc->onLocalDescription([weak](rtc::Description desc){ + auto s = weak.lock(); + if(not s or not s->ws or not s->ws->isOpen()){ + return; + } + + data<sch::SignalDescription> out; + out.get<"type">().set(desc.typeString()); + out.get<"sdp">().set(std::string{desc}); + + auto eo_text = encode_message<sch::SignalDescription>(out); + if(eo_text.is_error()){ + return; + } + s->ws->send(eo_text.get_value()); + }); + + sess->pc->onLocalCandidate([weak](rtc::Candidate cand){ + auto s = weak.lock(); + if(not s or not s->ws or not s->ws->isOpen()){ + return; + } + + data<sch::SignalCandidate> out; + out.get<"type">().set(std::string{sig::candidate}); + out.get<"candidate">().set(cand.candidate()); + out.get<"mid">().set(cand.mid()); + + auto eo_text = encode_message<sch::SignalCandidate>(out); + if(eo_text.is_error()){ + return; + } + s->ws->send(eo_text.get_value()); + }); + + sess->pc->onStateChange([this, weak](rtc::PeerConnection::State state){ + auto s = weak.lock(); + if(not s){ + return; + } + if(state == rtc::PeerConnection::State::Closed or + state == rtc::PeerConnection::State::Failed){ + drop_session(s->id); + } + }); + + /** + * Creating the channel here is what triggers negotiation: with automatic + * negotiation on, the offer is generated because there is now something to + * offer. Doing it from the orchestrator rather than waiting for the worker + * also fixes the ordering -- the channel is in the offer, so the worker + * never has to guess whether it exists yet. + */ + sess->dc = sess->pc->createDataChannel("control"); + + sess->dc->onOpen([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + if(on_open_){ + on_open_(s); + } + }); + + sess->dc->onMessage([this, weak](rtc::message_variant msg){ + auto s = weak.lock(); + if(not s){ + return; + } + const std::string text = text_of(msg); + if(text.empty()){ + return; + } + if(on_message_){ + on_message_(s, text); + } + }); + + sess->dc->onClosed([this, weak](){ + auto s = weak.lock(); + if(not s){ + return; + } + drop_session(s->id); + }); +} + +void orchestrator::handle_signal(const std::shared_ptr<session> &sess, + const std::string &text){ + // Two stage decode, same as the control protocol: learn the type without + // committing to a payload shape. + auto eo_hdr = decode_message<sch::SignalHeader>(text); + if(eo_hdr.is_error()){ + std::cerr << "[" << sess->id << "] malformed signalling message\n"; + return; + } + + const std::string type = eo_hdr.get_value().get<"type">().stl_string(); + + if(type == sig::answer or type == sig::offer){ + auto eo = decode_message<sch::SignalDescription>(text); + if(eo.is_error()){ + std::cerr << "[" << sess->id << "] bad " << type << "\n"; + return; + } + try { + sess->pc->setRemoteDescription( + rtc::Description{eo.get_value().get<"sdp">().stl_string(), type}); + } catch (const std::exception &e){ + std::cerr << "[" << sess->id << "] setRemoteDescription: " + << e.what() << "\n"; + } + return; + } + + if(type == sig::candidate){ + auto eo = decode_message<sch::SignalCandidate>(text); + if(eo.is_error()){ + std::cerr << "[" << sess->id << "] bad candidate\n"; + return; + } + try { + sess->pc->addRemoteCandidate( + rtc::Candidate{eo.get_value().get<"candidate">().stl_string(), + eo.get_value().get<"mid">().stl_string()}); + } catch (const std::exception &e){ + std::cerr << "[" << sess->id << "] addRemoteCandidate: " + << e.what() << "\n"; + } + return; + } + + std::cerr << "[" << sess->id << "] unknown signalling type: " << type << "\n"; +} + +void orchestrator::drop_session(const std::string &id){ + std::shared_ptr<session> sess; + + { + std::lock_guard<std::mutex> lock{mutex_}; + auto iter = sessions_.find(id); + if(iter == sessions_.end()){ + // Already dropped. Several teardown paths converge here -- socket + // closed, channel closed, peer connection failed -- and whichever + // arrives first wins. + return; + } + sess = iter->second; + sessions_.erase(iter); + } + + // Notify outside the lock: the handler may well want to send to other + // sessions, and holding the registry lock across user code is how this + // kind of thing deadlocks later. + if(on_closed_){ + on_closed_(sess); + } +} + +bool orchestrator::send(const std::shared_ptr<session> &sess, + const std::string &text){ + if(not sess or not sess->dc or not sess->dc->isOpen()){ + return false; + } + return sess->dc->send(text); +} + +std::shared_ptr<session> orchestrator::find_session(const std::string &id) const { + std::lock_guard<std::mutex> lock{mutex_}; + + auto iter = sessions_.find(id); + if(iter == sessions_.end()){ + return nullptr; + } + + return iter->second; +} + +std::size_t orchestrator::session_count() const { + std::lock_guard<std::mutex> lock{mutex_}; + return sessions_.size(); +} + +void orchestrator::for_each_session( + const std::function<void(const std::shared_ptr<session> &)> &fn) const { + std::lock_guard<std::mutex> lock{mutex_}; + for(const auto &entry : sessions_){ + fn(entry.second); + } +} + +void orchestrator::stop(){ + std::map<std::string, std::shared_ptr<session>> taken; + + { + std::lock_guard<std::mutex> lock{mutex_}; + taken.swap(sessions_); + } + + // Close outside the lock. Closing a peer connection runs callbacks, and + // some of those callbacks want the lock. + for(auto &entry : taken){ + auto &sess = entry.second; + if(sess->dc){ + sess->dc->close(); + } + if(sess->pc){ + sess->pc->close(); + } + if(sess->ws){ + sess->ws->close(); + } + } + + if(server_){ + server_->stop(); + server_.reset(); + } +} + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/orchestrator.hpp b/poc/web_swarm/orchestrator/c++/orchestrator.hpp new file mode 100644 index 0000000..83ce835 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/orchestrator.hpp @@ -0,0 +1,113 @@ +#pragma once + +#include "session.hpp" + +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <mutex> +#include <string> +#include <string_view> + +namespace kel { +namespace lbm { +namespace swarm { + +struct orchestrator_config { + uint16_t port = 8080u; + + /** + * STUN is not needed on localhost or a LAN -- host candidates are enough, + * and they are gathered without any server. The slot is here so NAT + * traversal is a configuration change later rather than a code change. + */ + std::string stun_url; +}; + +/** + * Accepts workers and holds their connections open. + * + * This is the first code in the repository with concurrency in it, so the rule + * is worth stating rather than inferring: libdatachannel owns the threads and + * delivers every callback on one of them. The only mutable state the + * orchestrator itself owns is the session map, and it is guarded by one mutex. + * Nothing else here is shared. Resist growing that. + * + * A deliberate non-feature: no static file serving. The page has to come from + * somewhere else (python3 -m http.server over the nix output is fine for + * development), because a WebSocket server is not an HTTP server and pretending + * otherwise would mean carrying a second protocol for no benefit. + */ +class orchestrator { +public: + /** + * Called with the session and the decoded text of each control message. + * Handling lives outside this class so the transport can be tested without + * a protocol and the protocol without a transport. + */ + using message_handler = + std::function<void(const std::shared_ptr<session> &, const std::string &)>; + + explicit orchestrator(orchestrator_config cfg); + ~orchestrator(); + + orchestrator(const orchestrator &) = delete; + orchestrator &operator=(const orchestrator &) = delete; + + /// Starts listening. Returns the port actually bound, which differs from + /// the requested one when the config asked for 0. + uint16_t listen(); + + void on_message(message_handler handler); + + /// Invoked once a session's control channel is open and usable. + void on_channel_open(std::function<void(const std::shared_ptr<session> &)> handler); + + /// Invoked when a session goes away, however it goes away. + void on_channel_closed(std::function<void(const std::shared_ptr<session> &)> handler); + + /// Sends text on a session's control channel. False if it is not open. + bool send(const std::shared_ptr<session> &sess, const std::string &text); + + /// Number of sessions currently held. Takes the lock. + std::size_t session_count() const; + + /// Runs `fn` for each session under the lock. Keep it short. + void for_each_session( + const std::function<void(const std::shared_ptr<session> &)> &fn) const; + + /** + * Resolves a session id to something sendable, or null if it has gone. + * + * The registry deals in ids alone so it can stay transport free, which + * leaves this as the one place that maps an id back to a connection. + */ + std::shared_ptr<session> find_session(const std::string &id) const; + + void stop(); + +private: + void accept_client(std::shared_ptr<rtc::WebSocket> ws); + void begin_negotiation(const std::shared_ptr<session> &sess); + void handle_signal(const std::shared_ptr<session> &sess, const std::string &text); + void drop_session(const std::string &id); + + std::string next_session_id(); + + orchestrator_config cfg_; + + std::unique_ptr<rtc::WebSocketServer> server_; + + mutable std::mutex mutex_; + std::map<std::string, std::shared_ptr<session>> sessions_; + uint64_t next_id_ = 0u; + + message_handler on_message_; + std::function<void(const std::shared_ptr<session> &)> on_open_; + std::function<void(const std::shared_ptr<session> &)> on_closed_; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel diff --git a/poc/web_swarm/orchestrator/c++/session.hpp b/poc/web_swarm/orchestrator/c++/session.hpp new file mode 100644 index 0000000..cdc6be3 --- /dev/null +++ b/poc/web_swarm/orchestrator/c++/session.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include <rtc/rtc.hpp> + +#include <memory> +#include <mutex> +#include <string> + +namespace kel { +namespace lbm { +namespace swarm { + +/** + * One worker's connection, from the signalling socket through to the control + * channel. + * + * Lifetime is the reason this is a type rather than three locals. libdatachannel + * hands callbacks a reference to nothing in particular, and a PeerConnection + * dropped at the end of the accept handler takes the whole negotiation with it. + * The orchestrator therefore holds a shared_ptr per session and the callbacks + * capture a weak_ptr, so a worker that disconnects mid-negotiation tears down + * cleanly instead of leaving a callback pointing at freed state. + * + * Threading: every member here is touched from libdatachannel's threads, not + * from main. The handles themselves are set once during setup and then only + * read, and the mutable state behind them is libdatachannel's own problem. The + * orchestrator's own mutable state lives in the registry, behind its mutex. + */ +struct session { + /// Assigned by the orchestrator, not the worker. Stable for the connection. + std::string id; + + /// Signalling. Stays open after negotiation so renegotiation is possible. + std::shared_ptr<rtc::WebSocket> ws; + + std::shared_ptr<rtc::PeerConnection> pc; + + /** + * The control channel. Created by the orchestrator rather than the worker + * so that channel-open ordering is deterministic: the offer already + * contains it, so by the time the worker answers it knows the channel is + * coming and does not have to race onDataChannel against its own send. + */ + std::shared_ptr<rtc::DataChannel> dc; +}; + +} // namespace swarm +} // namespace lbm +} // namespace kel |
