From a14896f9ed209dd3f9597722e5a5697bd7dbf531 Mon Sep 17 00:00:00 2001 From: "Claudius \"keldu\" Holeksa" Date: Mon, 4 Dec 2023 12:18:14 +0100 Subject: meta: Renamed folder containing source --- modules/SConscript | 8 + modules/async/.nix/derivation.nix | 28 + modules/async/SConscript | 38 + modules/async/SConstruct | 66 ++ modules/async/async.cpp | 419 +++++++++++ modules/async/async.h | 1023 +++++++++++++++++++++++++++ modules/async/async.tmpl.h | 767 ++++++++++++++++++++ modules/codec-json/.nix/derivation.nix | 30 + modules/codec-json/SConscript | 38 + modules/codec-json/SConstruct | 66 ++ modules/codec-json/json.h | 120 ++++ modules/codec-json/json.tmpl.h | 734 +++++++++++++++++++ modules/codec-minecraft/.nix/derivation.nix | 32 + modules/codec-minecraft/SConscript | 38 + modules/codec-minecraft/SConstruct | 66 ++ modules/codec-minecraft/minecraft.h | 116 +++ modules/codec-minecraft/var_int_data.cpp | 36 + modules/codec-minecraft/var_int_data.h | 31 + modules/codec-netcdf/.nix/derivation.nix | 33 + modules/codec-netcdf/SConscript | 38 + modules/codec-netcdf/SConstruct | 66 ++ modules/codec-netcdf/netcdf.h | 151 ++++ modules/codec-netcdf/netcdf.tmpl.h | 348 +++++++++ modules/codec/.nix/derivation.nix | 28 + modules/codec/SConscript | 38 + modules/codec/SConstruct | 66 ++ modules/codec/args.h | 123 ++++ modules/codec/csv.h | 25 + modules/codec/data.h | 370 ++++++++++ modules/codec/forst.h | 32 + modules/codec/forst.tmpl.h | 7 + modules/codec/interface.h | 103 +++ modules/codec/rpc.h | 26 + modules/codec/schema.h | 109 +++ modules/codec/schema_hash.h | 105 +++ modules/codec/schema_stringify.h | 118 +++ modules/codec/simple.h | 389 ++++++++++ modules/codec/stream_value.h | 64 ++ modules/core/.nix/derivation.nix | 21 + modules/core/SConscript | 42 ++ modules/core/SConstruct | 66 ++ modules/core/array.h | 96 +++ modules/core/buffer.cpp | 436 ++++++++++++ modules/core/buffer.h | 199 ++++++ modules/core/common.h | 75 ++ modules/core/error.cpp | 156 ++++ modules/core/error.h | 248 +++++++ modules/core/id.h | 54 ++ modules/core/id_map.h | 137 ++++ modules/core/mcts.h | 52 ++ modules/core/platonic.h | 103 +++ modules/core/string_literal.h | 57 ++ modules/core/templates.h | 150 ++++ modules/core/tree.h | 248 +++++++ modules/device-hip/.nix/derivation.nix | 30 + modules/device-hip/SConscript | 38 + modules/device-hip/SConstruct | 66 ++ modules/device-hip/rpc.h | 7 + modules/io-fs/.nix/derivation.nix | 33 + modules/io-fs/SConscript | 38 + modules/io-fs/SConstruct | 66 ++ modules/io-tls/.nix/derivation.nix | 32 + modules/io-tls/SConscript | 38 + modules/io-tls/SConstruct | 66 ++ modules/io-tls/tls.cpp | 252 +++++++ modules/io-tls/tls.h | 68 ++ modules/io/.nix/derivation.nix | 29 + modules/io/SConscript | 38 + modules/io/SConstruct | 66 ++ modules/io/io.cpp | 70 ++ modules/io/io.h | 219 ++++++ modules/io/io_helpers.cpp | 85 +++ modules/io/io_helpers.h | 53 ++ modules/io/io_unix.cpp | 894 +++++++++++++++++++++++ modules/io_codec/.nix/derivation.nix | 31 + modules/io_codec/SConscript | 38 + modules/io_codec/SConstruct | 66 ++ modules/io_codec/io_peer.h | 104 +++ modules/io_codec/io_peer.tmpl.h | 117 +++ modules/io_codec/rpc.h | 31 + modules/test/.nix/derivation.nix | 28 + modules/test/SConscript | 33 + modules/test/SConstruct | 66 ++ modules/test/suite.cpp | 107 +++ modules/test/suite.h | 43 ++ modules/tools/.nix/derivation.nix | 31 + modules/tools/SConscript | 32 + modules/tools/SConstruct | 71 ++ modules/tools/c_gen_iface.cpp | 11 + modules/tools/c_gen_iface.hpp | 535 ++++++++++++++ modules/tools/cli_analyzer.hpp | 114 +++ modules/window-opengl/.nix/derivation.nix | 37 + modules/window-opengl/SConscript | 38 + modules/window-opengl/SConstruct | 72 ++ modules/window-opengl/gl_backends.h | 10 + modules/window-opengl/gl_context.h | 43 ++ modules/window-opengl/gl_window.h | 8 + modules/window-opengl/gl_xcb.cpp | 269 +++++++ modules/window-opengl/gl_xcb.h | 68 ++ modules/window/.nix/derivation.nix | 34 + modules/window/SConscript | 38 + modules/window/SConstruct | 66 ++ modules/window/backends.h | 10 + modules/window/device.h | 18 + modules/window/linux_xcb.h | 5 + modules/window/old.dummy | 399 +++++++++++ modules/window/video_mode.h | 11 + modules/window/window.h | 79 +++ modules/window/xcb.cpp | 290 ++++++++ modules/window/xcb.h | 105 +++ 110 files changed, 13281 insertions(+) create mode 100644 modules/SConscript create mode 100644 modules/async/.nix/derivation.nix create mode 100644 modules/async/SConscript create mode 100644 modules/async/SConstruct create mode 100644 modules/async/async.cpp create mode 100644 modules/async/async.h create mode 100644 modules/async/async.tmpl.h create mode 100644 modules/codec-json/.nix/derivation.nix create mode 100644 modules/codec-json/SConscript create mode 100644 modules/codec-json/SConstruct create mode 100644 modules/codec-json/json.h create mode 100644 modules/codec-json/json.tmpl.h create mode 100644 modules/codec-minecraft/.nix/derivation.nix create mode 100644 modules/codec-minecraft/SConscript create mode 100644 modules/codec-minecraft/SConstruct create mode 100644 modules/codec-minecraft/minecraft.h create mode 100644 modules/codec-minecraft/var_int_data.cpp create mode 100644 modules/codec-minecraft/var_int_data.h create mode 100644 modules/codec-netcdf/.nix/derivation.nix create mode 100644 modules/codec-netcdf/SConscript create mode 100644 modules/codec-netcdf/SConstruct create mode 100644 modules/codec-netcdf/netcdf.h create mode 100644 modules/codec-netcdf/netcdf.tmpl.h create mode 100644 modules/codec/.nix/derivation.nix create mode 100644 modules/codec/SConscript create mode 100644 modules/codec/SConstruct create mode 100644 modules/codec/args.h create mode 100644 modules/codec/csv.h create mode 100644 modules/codec/data.h create mode 100644 modules/codec/forst.h create mode 100644 modules/codec/forst.tmpl.h create mode 100644 modules/codec/interface.h create mode 100644 modules/codec/rpc.h create mode 100644 modules/codec/schema.h create mode 100644 modules/codec/schema_hash.h create mode 100644 modules/codec/schema_stringify.h create mode 100644 modules/codec/simple.h create mode 100644 modules/codec/stream_value.h create mode 100644 modules/core/.nix/derivation.nix create mode 100644 modules/core/SConscript create mode 100644 modules/core/SConstruct create mode 100644 modules/core/array.h create mode 100644 modules/core/buffer.cpp create mode 100644 modules/core/buffer.h create mode 100644 modules/core/common.h create mode 100644 modules/core/error.cpp create mode 100644 modules/core/error.h create mode 100644 modules/core/id.h create mode 100644 modules/core/id_map.h create mode 100644 modules/core/mcts.h create mode 100644 modules/core/platonic.h create mode 100644 modules/core/string_literal.h create mode 100644 modules/core/templates.h create mode 100644 modules/core/tree.h create mode 100644 modules/device-hip/.nix/derivation.nix create mode 100644 modules/device-hip/SConscript create mode 100644 modules/device-hip/SConstruct create mode 100644 modules/device-hip/rpc.h create mode 100644 modules/io-fs/.nix/derivation.nix create mode 100644 modules/io-fs/SConscript create mode 100644 modules/io-fs/SConstruct create mode 100644 modules/io-tls/.nix/derivation.nix create mode 100644 modules/io-tls/SConscript create mode 100644 modules/io-tls/SConstruct create mode 100644 modules/io-tls/tls.cpp create mode 100644 modules/io-tls/tls.h create mode 100644 modules/io/.nix/derivation.nix create mode 100644 modules/io/SConscript create mode 100644 modules/io/SConstruct create mode 100644 modules/io/io.cpp create mode 100644 modules/io/io.h create mode 100644 modules/io/io_helpers.cpp create mode 100644 modules/io/io_helpers.h create mode 100644 modules/io/io_unix.cpp create mode 100644 modules/io_codec/.nix/derivation.nix create mode 100644 modules/io_codec/SConscript create mode 100644 modules/io_codec/SConstruct create mode 100644 modules/io_codec/io_peer.h create mode 100644 modules/io_codec/io_peer.tmpl.h create mode 100644 modules/io_codec/rpc.h create mode 100644 modules/test/.nix/derivation.nix create mode 100644 modules/test/SConscript create mode 100644 modules/test/SConstruct create mode 100644 modules/test/suite.cpp create mode 100644 modules/test/suite.h create mode 100644 modules/tools/.nix/derivation.nix create mode 100644 modules/tools/SConscript create mode 100644 modules/tools/SConstruct create mode 100644 modules/tools/c_gen_iface.cpp create mode 100644 modules/tools/c_gen_iface.hpp create mode 100644 modules/tools/cli_analyzer.hpp create mode 100644 modules/window-opengl/.nix/derivation.nix create mode 100644 modules/window-opengl/SConscript create mode 100644 modules/window-opengl/SConstruct create mode 100644 modules/window-opengl/gl_backends.h create mode 100644 modules/window-opengl/gl_context.h create mode 100644 modules/window-opengl/gl_window.h create mode 100644 modules/window-opengl/gl_xcb.cpp create mode 100644 modules/window-opengl/gl_xcb.h create mode 100644 modules/window/.nix/derivation.nix create mode 100644 modules/window/SConscript create mode 100644 modules/window/SConstruct create mode 100644 modules/window/backends.h create mode 100644 modules/window/device.h create mode 100644 modules/window/linux_xcb.h create mode 100644 modules/window/old.dummy create mode 100644 modules/window/video_mode.h create mode 100644 modules/window/window.h create mode 100644 modules/window/xcb.cpp create mode 100644 modules/window/xcb.h (limited to 'modules') diff --git a/modules/SConscript b/modules/SConscript new file mode 100644 index 0000000..8da5a3d --- /dev/null +++ b/modules/SConscript @@ -0,0 +1,8 @@ +#!/bin/false + +Import('env') + +# Export to other libs +Export('env'); +SConscript('core/SConscript'); +SConscript('async/SConscript'); diff --git a/modules/async/.nix/derivation.nix b/modules/async/.nix/derivation.nix new file mode 100644 index 0000000..aad258f --- /dev/null +++ b/modules/async/.nix/derivation.nix @@ -0,0 +1,28 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-async"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/async/SConscript b/modules/async/SConscript new file mode 100644 index 0000000..69f8950 --- /dev/null +++ b/modules/async/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +async_env = env.Clone(); + +async_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +async_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += async_env.sources; +env.headers += async_env.headers; + +## Shared lib +objects_shared = [] +async_env.add_source_files(objects_shared, async_env.sources, shared=True); +async_env.library_shared = async_env.SharedLibrary('#build/forstio-async', [objects_shared]); + +## Static lib +objects_static = [] +async_env.add_source_files(objects_static, async_env.sources, shared=False); +async_env.library_static = async_env.StaticLibrary('#build/forstio-async', [objects_static]); + +# Set Alias +env.Alias('library_async', [async_env.library_shared, async_env.library_static]); + +env.targets += ['library_async']; + +# Install +env.Install('$prefix/lib/', [async_env.library_shared, async_env.library_static]); +env.Install('$prefix/include/forstio/async/', [async_env.headers]); diff --git a/modules/async/SConstruct b/modules/async/SConstruct new file mode 100644 index 0000000..0d7b7c6 --- /dev/null +++ b/modules/async/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-core']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/async/async.cpp b/modules/async/async.cpp new file mode 100644 index 0000000..360e455 --- /dev/null +++ b/modules/async/async.cpp @@ -0,0 +1,419 @@ +#include "async.h" +#include +#include + +#include +#include + +namespace saw { +namespace { +thread_local event_loop *local_loop = nullptr; + +event_loop ¤t_event_loop() { + event_loop *loop = local_loop; + assert(loop); + return *loop; +} +} // namespace + +conveyor_node::conveyor_node() {} + +conveyor_node_with_child_mixin::conveyor_node_with_child_mixin( + own &&child_, conveyor_node &owner) + : child{std::move(child_)} { + assert(child); + + child->notify_parent_attached(owner); +} + +error_or> +conveyor_node_with_child_mixin::swap_child(own &&swapee) { + SAW_ASSERT(child) { + return make_error("Child should exist if this function is called"); + } + own old_child = std::move(child); + + /** + * We need the parent of the old_child's next storage + */ + conveyor_storage *old_storage = old_child->next_storage(); + conveyor_storage *old_storage_parent = + old_storage ? old_storage->get_parent() : nullptr; + + /** + * Swap in the new child + */ + if (swapee) { + child = std::move(swapee); + + /** + * Then we need to set the new child's storage parent since the next + * storage has a nullptr set And if the old_storage_parent is a nullptr, + * then it doesn't matter. So we don't check for it + */ + conveyor_storage *swapee_storage = child->next_storage(); + if (swapee_storage) { + swapee_storage->set_parent(old_storage_parent); + } + } + + return old_child; +} + +conveyor_storage::conveyor_storage() {} + +conveyor_storage::~conveyor_storage() {} + +conveyor_storage *conveyor_storage::get_parent() const { return parent_; } + +void conveyor_event_storage::set_parent(conveyor_storage *p) { + /* + * parent check isn't needed, but is used + * for the assert, because the storage should + * be armed if there was an element present + * and a valid parent + */ + if (/*!parent && */ p && !is_armed() && queued() > 0) { + assert(!parent_); + if (p->space() > 0) { + arm_later(); + } + } + + parent_ = p; +} + +conveyor_event_storage::conveyor_event_storage() : conveyor_storage{} {} + +conveyor_base::conveyor_base(own &&node_p) + : node_{std::move(node_p)} {} + +error propagate_error::operator()(const error &error) const { + return error.copy_error(); +} + +error propagate_error::operator()(error &&err) { return std::move(err); } + +event::event() : event(current_event_loop()) {} + +event::event(event_loop &loop) : loop_{loop} {} + +event::~event() { disarm(); } + +void event::arm_next() { + assert(&loop_ == local_loop); + if (prev_ == nullptr) { + // Push the next_insert_point back by one + // and inserts itself before that + next_ = *loop_.next_insert_point_; + prev_ = loop_.next_insert_point_; + *prev_ = this; + if (next_) { + next_->prev_ = &next_; + } + + // Set the new insertion ptr location to next + loop_.next_insert_point_ = &next_; + + // Pushes back the later insert point if it was pointing at the + // previous event + if (loop_.later_insert_point_ == prev_) { + loop_.later_insert_point_ = &next_; + } + + // If tail_ points at the same location then + // we are at the end and have to update tail_ then. + // Technically should be possible by checking if + // next is a `nullptr` + if (loop_.tail_ == prev_) { + loop_.tail_ = &next_; + } + + loop_.set_runnable(true); + } +} + +void event::arm_later() { + assert(&loop_ == local_loop); + + if (prev_ == nullptr) { + next_ = *loop_.later_insert_point_; + prev_ = loop_.later_insert_point_; + *prev_ = this; + if (next_) { + next_->prev_ = &next_; + } + + loop_.later_insert_point_ = &next_; + if (loop_.tail_ == prev_) { + loop_.tail_ = &next_; + } + + loop_.set_runnable(true); + } +} + +void event::arm_last() { + assert(&loop_ == local_loop); + + if (prev_ == nullptr) { + next_ = *loop_.later_insert_point_; + prev_ = loop_.later_insert_point_; + *prev_ = this; + if (next_) { + next_->prev_ = &next_; + } + + if (loop_.tail_ == prev_) { + loop_.tail_ = &next_; + } + + loop_.set_runnable(true); + } +} + +void event::disarm() { + if (prev_ != nullptr) { + if (loop_.tail_ == &next_) { + loop_.tail_ = prev_; + } + + if (loop_.next_insert_point_ == &next_) { + loop_.next_insert_point_ = prev_; + } + + *prev_ = next_; + if (next_) { + next_->prev_ = prev_; + } + + prev_ = nullptr; + next_ = nullptr; + } +} + +bool event::is_armed() const { return prev_ != nullptr; } + +conveyor_sink::conveyor_sink() : node_{nullptr} {} + +conveyor_sink::conveyor_sink(own &&node_p) + : node_{std::move(node_p)} {} + +void event_loop::set_runnable(bool runnable) { is_runnable_ = runnable; } + +event_loop::event_loop() {} + +event_loop::event_loop(own &&ep) + : event_port_{std::move(ep)} {} + +event_loop::~event_loop() { assert(local_loop != this); } + +void event_loop::enter_scope() { + assert(!local_loop); + local_loop = this; +} + +void event_loop::leave_scope() { + assert(local_loop == this); + local_loop = nullptr; +} + +bool event_loop::turn_loop() { + size_t turn_step = 0; + while (head_ && turn_step < 65536) { + if (!turn()) { + return false; + } + ++turn_step; + } + return true; +} + +bool event_loop::turn() { + event *event = head_; + + if (!event) { + return false; + } + + head_ = event->next_; + if (head_) { + head_->prev_ = &head_; + } + + next_insert_point_ = &head_; + if (later_insert_point_ == &event->next_) { + later_insert_point_ = &head_; + } + if (tail_ == &event->next_) { + tail_ = &head_; + } + + event->next_ = nullptr; + event->prev_ = nullptr; + + next_insert_point_ = &head_; + + event->fire(); + + return true; +} + +bool event_loop::wait(const std::chrono::steady_clock::duration &duration) { + if (event_port_) { + event_port_->wait(duration); + } + + return turn_loop(); +} + +bool event_loop::wait(const std::chrono::steady_clock::time_point &time_point) { + if (event_port_) { + event_port_->wait(time_point); + } + + return turn_loop(); +} + +bool event_loop::wait() { + if (event_port_) { + event_port_->wait(); + } + + return turn_loop(); +} + +bool event_loop::poll() { + if (event_port_) { + event_port_->poll(); + } + + return turn_loop(); +} + +event_port *event_loop::get_event_port() { return event_port_.get(); } + +conveyor_sink_set &event_loop::daemon() { + if (!daemon_sink_) { + daemon_sink_ = heap(); + } + return *daemon_sink_; +} + +wait_scope::wait_scope(event_loop &loop) : loop_{loop} { loop_.enter_scope(); } + +wait_scope::~wait_scope() { loop_.leave_scope(); } + +void wait_scope::wait() { loop_.wait(); } + +void wait_scope::wait(const std::chrono::steady_clock::duration &duration) { + loop_.wait(duration); +} + +void wait_scope::wait(const std::chrono::steady_clock::time_point &time_point) { + loop_.wait(time_point); +} + +void wait_scope::poll() { loop_.poll(); } + +error_or> +convert_conveyor_node_base::swap_child(own &&swapee) noexcept { + return child_mixin_.swap_child(std::move(swapee)); +} + +conveyor_storage *convert_conveyor_node_base::next_storage() noexcept { + if (!child_mixin_.child) { + return nullptr; + } + return child_mixin_.child->next_storage(); +} + +immediate_conveyor_node_base::immediate_conveyor_node_base() + : conveyor_event_storage{} {} + +merge_conveyor_node_base::merge_conveyor_node_base() + : conveyor_event_storage{} {} + +error_or> queue_buffer_conveyor_node_base::swap_child( + own &&swapee_) noexcept { + return child_mixin_.swap_child(std::move(swapee_)); +} + +void conveyor_sink_set::destroy_sink_conveyor_node(conveyor_node &node) { + if (!is_armed()) { + arm_last(); + } + + delete_nodes_.push(&node); +} + +void conveyor_sink_set::fail(error &&error) { + /// @todo call error_handler +} + +conveyor_sink_set::conveyor_sink_set(event_loop &event_loop) + : event{event_loop} {} + +void conveyor_sink_set::add(conveyor &&sink) { + auto nas = conveyor::from_conveyor(std::move(sink)); + SAW_ASSERT(nas) { return; } + conveyor_storage *storage = nas->next_storage(); + + own sink_node = nullptr; + try { + sink_node = heap(std::move(nas), *this); + } catch (std::bad_alloc &) { + return; + } + if (storage) { + storage->set_parent(sink_node.get()); + } + + sink_nodes_.emplace_back(std::move(sink_node)); +} + +void conveyor_sink_set::fire() { + while (!delete_nodes_.empty()) { + conveyor_node *node = delete_nodes_.front(); + /*auto erased = */ std::remove_if(sink_nodes_.begin(), + sink_nodes_.end(), + [node](own &element) { + return node == element.get(); + }); + delete_nodes_.pop(); + } +} + +convert_conveyor_node_base::convert_conveyor_node_base(own &&dep) + : child_mixin_{std::move(dep), *this} {} + +void convert_conveyor_node_base::get_result(error_or_value &err_or_val) { + get_impl(err_or_val); +} + +void attach_conveyor_node_base::get_result( + error_or_value &err_or_val) noexcept { + if (child_mixin_.child) { + child_mixin_.child->get_result(err_or_val); + } +} + +error_or> +attach_conveyor_node_base::swap_child(own &&swapee_) noexcept { + return child_mixin_.swap_child(std::move(swapee_)); +} + +conveyor_storage *attach_conveyor_node_base::next_storage() noexcept { + if (!child_mixin_.child) { + return nullptr; + } + + return child_mixin_.child->next_storage(); +} + +void detach_conveyor(conveyor &&conveyor) { + event_loop &loop = current_event_loop(); + conveyor_sink_set &sink = loop.daemon(); + sink.add(std::move(conveyor)); +} +} // namespace saw diff --git a/modules/async/async.h b/modules/async/async.h new file mode 100644 index 0000000..8190be0 --- /dev/null +++ b/modules/async/async.h @@ -0,0 +1,1023 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace saw { +class conveyor_storage; +class conveyor_node { +public: + conveyor_node(); + virtual ~conveyor_node() = default; + + /** + * Internal method to retrieve results from children + */ + virtual void get_result(error_or_value &err_or_val) = 0; + + /** + * Swap out child with another one + */ + virtual error_or> + swap_child(own &&swapee_) = 0; + + /** + * Retrieve the next storage node + */ + virtual conveyor_storage *next_storage() = 0; + + /** + * Notify that a new parent was attached + * Only relevant for the feeding nodes + */ + virtual void notify_parent_attached(conveyor_node &){}; +}; + +class conveyor_node_with_child_mixin final { +public: + own child = nullptr; + + conveyor_node_with_child_mixin(own &&child_, + conveyor_node &owner_); + ~conveyor_node_with_child_mixin() = default; + + /** + * Swap out children and return the child ptr, since the caller is the child + * itself. Stack needs to be cleared before the child is destroyed, so the + * swapped out node is returned as well. + */ + error_or> swap_child(own &&swapee); +}; + +class conveyor_node_with_parent_mixin final { +public: + conveyor_node *parent = nullptr; + + error_or> + swap_child_of_parent(own &&swapee) { + SAW_ASSERT(parent) { + return make_error( + "Can't swap child, because parent doesn't exist"); + } + + return parent->swap_child(std::move(swapee)); + } + void change_parent(conveyor_node *p) { parent = p; } +}; + +class event_loop; +class wait_scope; +/* + * Event class similar to capn'proto. + * https://github.com/capnproto/capnproto + */ +class event { +private: + event_loop &loop_; + event **prev_ = nullptr; + event *next_ = nullptr; + + friend class event_loop; + +public: + event(); + event(event_loop &loop); + virtual ~event(); + + virtual void fire() = 0; + + void arm_next(); + void arm_later(); + void arm_last(); + void disarm(); + + bool is_armed() const; +}; + +class conveyor_storage { +protected: + conveyor_storage *parent_ = nullptr; + +public: + conveyor_storage(); + virtual ~conveyor_storage(); + + virtual size_t space() const = 0; + virtual size_t queued() const = 0; + virtual void child_has_fired() = 0; + virtual void parent_has_fired() = 0; + + virtual void set_parent(conveyor_storage *parent) = 0; + conveyor_storage *get_parent() const; +}; + +class conveyor_event_storage : public conveyor_storage, public event { +public: + conveyor_event_storage(); + virtual ~conveyor_event_storage() = default; + + void set_parent(conveyor_storage *parent) override; +}; + +class conveyor_base { +protected: + own node_; + +public: + conveyor_base(own &&node_p); + virtual ~conveyor_base() = default; + + conveyor_base(conveyor_base &&) = default; + conveyor_base &operator=(conveyor_base &&) = default; + + void get(error_or_value &err_or_val); +}; + +template class conveyor; + +template conveyor chained_conveyor_type(T *); + +// template Conveyor chainedConveyorType(Conveyor *); + +template T remove_error_or_type(T *); + +template T remove_error_or_type(error_or *); + +template +using remove_error_or = decltype(remove_error_or_type((T *)nullptr)); + +template +using chained_conveyors = decltype(chained_conveyor_type((T *)nullptr)); + +template +using conveyor_result = + chained_conveyors>>; + +struct propagate_error { +public: + error operator()(const error &err) const; + error operator()(error &&err); +}; + +class conveyor_sink { +private: + own node_; + +public: + conveyor_sink(); + conveyor_sink(own &&node); + + conveyor_sink(conveyor_sink &&) = default; + conveyor_sink &operator=(conveyor_sink &&) = default; +}; + +template class merge_conveyor_node_data; + +template class merge_conveyor { +private: + lent> data_; + +public: + merge_conveyor() = default; + merge_conveyor(lent> d); + ~merge_conveyor(); + + void attach(conveyor conv); +}; + +/** + * Main interface for async operations. + */ +template class conveyor final : public conveyor_base { +public: + /** + * Construct an immediately fulfilled node + */ + conveyor(fix_void value); + + /** + * Construct an immediately failed node + */ + conveyor(error &&err); + + /** + * Construct a conveyor with a child node + */ + conveyor(own node_p); + + conveyor(conveyor &&) = default; + conveyor &operator=(conveyor &&) = default; + + /** + * This method converts values or errors from children + */ + template + [[nodiscard]] conveyor_result + then(Func &&func, ErrorFunc &&error_func = propagate_error()); + + /** + * This method adds a buffer node in the conveyor chains which acts as a + * scheduler interrupt point and collects elements up to the supplied limit. + */ + [[nodiscard]] conveyor + buffer(size_t limit = std::numeric_limits::max()); + + /** + * This method just takes ownership of any supplied types, + * which are destroyed when the chain gets destroyed. + * Useful for resource lifetime control. + */ + template + [[nodiscard]] conveyor attach(Args &&...args); + + /** @todo implement + * This method limits the total amount of passed elements + * Be careful where you place this node into the chain. + * If you meant to fork it and destroy paths you shouldn't place + * an interrupt point between the fork and this limiter + */ + [[nodiscard]] conveyor limit(size_t val = 1); + + /** + * + */ + [[nodiscard]] std::pair, merge_conveyor> merge(); + + /** + * Moves the conveyor chain into a thread local storage point which drops + * every element. Use sink() if you want to control the lifetime of a + * conveyor chain + */ + template + void detach(ErrorFunc &&err_func = propagate_error()); + /** + * Creates a local sink which drops elements, but lifetime control remains + * in your hand. + */ + template + [[nodiscard]] conveyor_sink + sink(ErrorFunc &&error_func = propagate_error()); + + /** + * If no sink() or detach() is used you have to take elements out of the + * chain yourself. + */ + error_or> take(); + + /** @todo implement + * Specifically pump elements through this chain with the provided + * wait_scope + */ + void poll(wait_scope &wait_scope); + + // helper + static conveyor to_conveyor(own node); + + // helper + static own from_conveyor(conveyor conveyor); +}; + +template conveyor_result exec_later(Func &&func); + +/* + * Join Conveyors into a single one + */ +template +conveyor> +join_conveyors(std::tuple...> &conveyors); + +template class conveyor_feeder { +public: + virtual ~conveyor_feeder() = default; + + virtual void feed(T &&data) = 0; + virtual void fail(error &&error) = 0; + + virtual size_t space() const = 0; + virtual size_t queued() const = 0; + + virtual error swap(conveyor &&conveyor) noexcept = 0; +}; + +template <> class conveyor_feeder { +public: + virtual ~conveyor_feeder() = default; + + virtual void feed(void_t &&value = void_t{}) = 0; + virtual void fail(error &&error) = 0; + + virtual size_t space() const = 0; + virtual size_t queued() const = 0; + + virtual error swap(conveyor &&conveyor) noexcept = 0; +}; + +template struct conveyor_and_feeder { + own> feeder; + class conveyor conveyor; +}; + +template conveyor_and_feeder new_conveyor_and_feeder(); + +template conveyor_and_feeder one_time_conveyor_and_feeder(); + +enum class Signal : uint8_t { Terminate, User1 }; + +/** + * Class which acts as a correspondent between the running framework and outside + * events which may be signals from the operating system or just other threads. + * Default EventPorts are supplied by setupAsyncIo() in io.h + */ +class event_port { +public: + virtual ~event_port() = default; + + virtual conveyor on_signal(Signal signal) = 0; + + virtual void poll() = 0; + virtual void wait() = 0; + virtual void wait(const std::chrono::steady_clock::duration &) = 0; + virtual void wait(const std::chrono::steady_clock::time_point &) = 0; + + virtual void wake() = 0; +}; + +class sink_conveyor_node; + +class conveyor_sink_set final : public event { +private: + /* + class Helper final : public Event { + private: + void destroySinkConveyorNode(ConveyorNode& sink); + void fail(Error&& error); + + std::vector> sink_nodes; + std::queue delete_nodes; + std::function error_handler; + + public: + ConveyorSinks() = default; + ConveyorSinks(EventLoop& event_loop); + + void add(Conveyor node); + + void fire() override {} + }; + + gin::Own helper; + */ + friend class sink_conveyor_node; + + void destroy_sink_conveyor_node(conveyor_node &sink_node); + void fail(error &&err); + + std::list> sink_nodes_; + + std::queue delete_nodes_; + + std::function error_handler_; + +public: + // ConveyorSinks(); + // ConveyorSinks(EventLoop& event_loop); + conveyor_sink_set() = default; + conveyor_sink_set(event_loop &event_loop); + + void add(conveyor &&node); + + void fire() override; +}; + +/* + * EventLoop class similar to capn'proto. + * https://github.com/capnproto/capnproto + */ +class event_loop { +private: + friend class event; + event *head_ = nullptr; + event **tail_ = &head_; + event **next_insert_point_ = &head_; + event **later_insert_point_ = &head_; + + bool is_runnable_ = false; + + own event_port_ = nullptr; + + own daemon_sink_ = nullptr; + + // functions + void set_runnable(bool runnable); + + friend class wait_scope; + void enter_scope(); + void leave_scope(); + + bool turn_loop(); + bool turn(); + +public: + event_loop(); + event_loop(own &&port); + ~event_loop(); + + event_loop(event_loop &&) = default; + event_loop &operator=(event_loop &&) = default; + + bool wait(); + bool wait(const std::chrono::steady_clock::duration &); + bool wait(const std::chrono::steady_clock::time_point &); + bool poll(); + + event_port *get_event_port(); + + conveyor_sink_set &daemon(); +}; + +/* + * WaitScope class similar to capn'proto. + * https://github.com/capnproto/capnproto + */ +class wait_scope { +private: + event_loop &loop_; + +public: + wait_scope(event_loop &loop); + ~wait_scope(); + + void wait(); + void wait(const std::chrono::steady_clock::duration &); + void wait(const std::chrono::steady_clock::time_point &); + void poll(); +}; + +template conveyor_result yield_next(Func &&func); + +template conveyor_result yield_later(Func &&func); + +template conveyor_result yield_last(Func &&func); +} // namespace saw + +// Secret stuff +// Aka private semi hidden classes +namespace saw { + +template struct fix_void_caller { + template static Out apply(Func &func, In &&in) { + return func(std::move(in)); + } +}; + +template struct fix_void_caller { + template static Out apply(Func &func, void_t &&in) { + (void)in; + return func(); + } +}; + +template struct fix_void_caller { + template static void_t apply(Func &func, In &&in) { + func(std::move(in)); + return void_t{}; + } +}; + +template <> struct fix_void_caller { + template static void_t apply(Func &func, void_t &&in) { + (void)in; + func(); + return void_t{}; + } +}; + +template class adapt_conveyor_node; + +template +class adapt_conveyor_feeder final : public conveyor_feeder> { +private: + adapt_conveyor_node *feedee_ = nullptr; + +public: + ~adapt_conveyor_feeder(); + + void set_feedee(adapt_conveyor_node *feedee); + + void feed(T &&value) override; + void fail(error &&error) override; + + size_t space() const override; + size_t queued() const override; + + error swap(conveyor &&conv) noexcept override; +}; + +template +class adapt_conveyor_node final : public conveyor_node, + public conveyor_event_storage { +private: + adapt_conveyor_feeder *feeder_ = nullptr; + + std::queue>> storage_; + + conveyor_node_with_parent_mixin parent_node_; + +public: + adapt_conveyor_node(); + ~adapt_conveyor_node(); + + void set_feeder(adapt_conveyor_feeder *feeder); + + void feed(T &&value); + void fail(error &&error); + + // ConveyorNode + void get_result(error_or_value &err_or_val) override; + + error_or> + swap_child(own &&swapee) noexcept override; + + conveyor_storage *next_storage() noexcept override; + void notify_parent_attached(conveyor_node &) noexcept override; + + // ConveyorStorage + size_t space() const override; + size_t queued() const override; + + void child_has_fired() override; + void parent_has_fired() override; + + // Event + void fire() override; +}; + +template class one_time_conveyor_node; + +template +class one_time_conveyor_feeder final : public conveyor_feeder> { +private: + one_time_conveyor_node *feedee_ = nullptr; + +public: + ~one_time_conveyor_feeder(); + + void set_feedee(one_time_conveyor_node *feedee); + + void feed(T &&value) override; + void fail(error &&error) override; + + size_t space() const override; + size_t queued() const override; +}; + +template +class one_time_conveyor_node final : public conveyor_node, + public conveyor_storage, + public event { +private: + one_time_conveyor_feeder *feeder_ = nullptr; + + bool passed_ = false; + maybe> storage_ = std::nullopt; + +public: + ~one_time_conveyor_node(); + + void set_feeder(one_time_conveyor_feeder *feeder); + + void feed(T &&value); + void fail(error &&error); + + // ConveyorNode + void get_result(error_or_value &err_or_val) override; + + error_or> + swap_child(own &&swapee) override; + + // ConveyorStorage + size_t space() const override; + size_t queued() const override; + + void child_has_fired() override {} + void parent_has_fired() override; + + // Event + void fire() override; +}; + +/** + * This class buffers and saves incoming elements and acts as an interrupt node + * for processing calls + */ +class queue_buffer_conveyor_node_base : public conveyor_node, + public conveyor_event_storage { +protected: + conveyor_node_with_child_mixin child_mixin_; + +public: + queue_buffer_conveyor_node_base(own child_) + : conveyor_event_storage{}, child_mixin_{std::move(child_), *this} {} + virtual ~queue_buffer_conveyor_node_base() = default; + + /** + * Use mixin + */ + error_or> + swap_child(own &&swapee_) noexcept override; + + conveyor_storage *next_storage() noexcept override { + return static_cast(this); + } +}; + +template +class queue_buffer_conveyor_node final + : public queue_buffer_conveyor_node_base { +private: + std::queue> storage_; + size_t max_store_; + +public: + queue_buffer_conveyor_node(own dep, size_t max_size) + : queue_buffer_conveyor_node_base{std::move(dep)}, max_store_{ + max_size} {} + // Event + void fire() override; + // ConveyorNode + void get_result(error_or_value &eov) noexcept override; + + // ConveyorStorage + size_t space() const override; + size_t queued() const override; + + void child_has_fired() override; + void parent_has_fired() override; +}; + +class attach_conveyor_node_base : public conveyor_node { +protected: + conveyor_node_with_child_mixin child_mixin_; + +public: + attach_conveyor_node_base(own &&child_) + : child_mixin_{std::move(child_), *this} {} + + virtual ~attach_conveyor_node_base() = default; + + void get_result(error_or_value &err_or_val) noexcept override; + + /** + * Use mixin + */ + error_or> + swap_child(own &&swapee_) noexcept override; + + conveyor_storage *next_storage() noexcept override; +}; + +template +class attach_conveyor_node final : public attach_conveyor_node_base { +public: + attach_conveyor_node(own &&dep, Args &&...args) + : attach_conveyor_node_base(std::move(dep)), attached_data_{ + std::move(args...)} {} + +private: + std::tuple attached_data_; +}; + +class convert_conveyor_node_base : public conveyor_node { +public: + convert_conveyor_node_base(own &&dep); + virtual ~convert_conveyor_node_base() = default; + + void get_result(error_or_value &err_or_val) override; + + virtual void get_impl(error_or_value &err_or_val) = 0; + + error_or> + swap_child(own &&swapee) noexcept override; + + conveyor_storage *next_storage() noexcept override; + +protected: + conveyor_node_with_child_mixin child_mixin_; +}; + +template +class convert_conveyor_node final : public convert_conveyor_node_base { +private: + Func func_; + ErrorFunc error_func_; + + static_assert(std::is_same>::value, + "Should never be of type ErrorOr"); + +public: + convert_conveyor_node(own &&dep, Func &&func, + ErrorFunc &&error_func) + : convert_conveyor_node_base(std::move(dep)), func_{std::move(func)}, + error_func_{std::move(error_func)} {} + + void get_impl(error_or_value &err_or_val) noexcept override { + error_or> dep_eov; + error_or>> &eov = + err_or_val.as>>(); + if (child_mixin_.child) { + child_mixin_.child->get_result(dep_eov); + if (dep_eov.is_value()) { + try { + + eov = fix_void_caller::apply( + func_, std::move(dep_eov.get_value())); + } catch (const std::bad_alloc &) { + eov = make_error("Out of memory"); + } catch (const std::exception &) { + eov = make_error( + "Exception in chain occured. Return ErrorOr if you " + "want to handle errors which are recoverable"); + } + } else if (dep_eov.is_error()) { + eov = error_func_(std::move(dep_eov.get_error())); + } else { + eov = make_error("No value set in dependency"); + } + } else { + eov = make_error("Conveyor doesn't have child"); + } + } +}; + +class sink_conveyor_node final : public conveyor_node, + public conveyor_event_storage { +private: + conveyor_node_with_child_mixin child_mixin_; + conveyor_sink_set *conveyor_sink_; + +public: + sink_conveyor_node(own node, conveyor_sink_set &conv_sink) + : conveyor_event_storage{}, child_mixin_{std::move(node), *this}, + conveyor_sink_{&conv_sink} {} + + sink_conveyor_node(own node) + : conveyor_event_storage{}, child_mixin_{std::move(node), *this}, + conveyor_sink_{nullptr} {} + + // Event only queued if a critical error occured + void fire() override { + // Queued for destruction of children, because this acts as a sink and + // no other event should be here + child_mixin_.child = nullptr; + + if (conveyor_sink_) { + conveyor_sink_->destroy_sink_conveyor_node(*this); + conveyor_sink_ = nullptr; + } + } + + // ConveyorStorage + size_t space() const override { return 1; } + size_t queued() const override { return 0; } + + // ConveyorNode + void get_result(error_or_value &err_or_val) noexcept override { + err_or_val.as() = + make_error("In a sink node no result can be returned"); + } + + error_or> + swap_child(own &&swapee) noexcept override { + return child_mixin_.swap_child(std::move(swapee)); + } + + // ConveyorStorage + void child_has_fired() override { + if (child_mixin_.child) { + error_or dep_eov; + child_mixin_.child->get_result(dep_eov); + if (dep_eov.is_error()) { + if (dep_eov.get_error().is_critical()) { + if (!is_armed()) { + arm_last(); + } + } + if (conveyor_sink_) { + conveyor_sink_->fail(std::move(dep_eov.get_error())); + } + } + } + } + + /* + * No parent needs to be fired since we always have space + */ + void parent_has_fired() override {} + + conveyor_storage *next_storage() override { + // Should never happen though + assert(false); + return nullptr; + // return static_cast(this); + } +}; + +class immediate_conveyor_node_base : public conveyor_node, + public conveyor_event_storage { +private: +public: + immediate_conveyor_node_base(); + + virtual ~immediate_conveyor_node_base() = default; + + error_or> + swap_child(own &&swapee) noexcept override { + (void)swapee; + return make_error("Node doesn't support swapping"); + } + + conveyor_storage *next_storage() noexcept override { + return static_cast(this); + } +}; + +template +class immediate_conveyor_node final : public immediate_conveyor_node_base { +private: + error_or> value_; + uint8_t retrieved_; + +public: + immediate_conveyor_node(fix_void &&val); + immediate_conveyor_node(error &&error); + + // ConveyorStorage + size_t space() const override; + size_t queued() const override; + + void child_has_fired() override; + void parent_has_fired() override; + + // ConveyorNode + void get_result(error_or_value &err_or_val) noexcept override { + if (retrieved_ > 0) { + err_or_val.as>() = + make_error("Already taken value"); + } else { + err_or_val.as>() = std::move(value_); + } + if (queued() > 0) { + ++retrieved_; + } + } + + // Event + void fire() override; +}; + +/* + * Collects every incoming value and throws it in one lane + */ +class merge_conveyor_node_base : public conveyor_node, + public conveyor_event_storage { +public: + merge_conveyor_node_base(); + + virtual ~merge_conveyor_node_base() = default; + + conveyor_storage *next_storage() noexcept override { + return static_cast(this); + } +}; + +template +class merge_conveyor_node : public merge_conveyor_node_base { +private: + class appendage final : public conveyor_node, public conveyor_storage { + public: + own child; + merge_conveyor_node *merger; + + maybe>> error_or_value_; + + public: + appendage(own n, merge_conveyor_node &m) + : conveyor_storage{}, child{std::move(n)}, merger{&m}, + error_or_value_{std::nullopt} {} + + bool child_storage_has_element_queued() const { + if (!child) { + return false; + } + conveyor_storage *storage = child->next_storage(); + if (storage) { + return storage->queued() > 0; + } + return false; + } + + void get_appendage_result(error_or_value &eov); + + /** + * ConveyorNode + */ + error_or> + swap_child(own &&swapee_) override; + + conveyor_storage *next_storage() noexcept override { + return static_cast(this); + } + + void get_result(error_or_value &err_or_val) override; + + /** + * ConveyorStorage + */ + size_t space() const override; + + size_t queued() const override; + + void child_has_fired() override; + + void parent_has_fired() override; + + void set_parent(conveyor_storage *par) override; + }; + + friend class merge_conveyor_node_data; + friend class appendage; + + our> data_; + size_t next_appendage_ = 0; + +public: + merge_conveyor_node(our> data); + ~merge_conveyor_node(); + // ConveyorNode + error_or> + swap_child(own &&c) noexcept override; + + // Event + void get_result(error_or_value &err_or_val) noexcept override; + + void fire() override; + + // ConveyorStorage + size_t space() const override; + size_t queued() const override; + void child_has_fired() override; + void parent_has_fired() override; +}; + +template class merge_conveyor_node_data { +public: + std::vector::appendage>> appendages; + + merge_conveyor_node *merger = nullptr; + +public: + void attach(conveyor conv); + + void governing_node_destroyed(); +}; + +/* +class JoinConveyorNodeBase : public ConveyorNode, public ConveyorEventStorage { +private: + +public: +}; + +template +class JoinConveyorNode final : public JoinConveyorNodeBase { +private: + template + class Appendage : public ConveyorEventStorage { + private: + Maybe data = std::nullopt; + + public: + size_t space() const override; + size_t queued() const override; + + void fire() override; + void get_result(ErrorOrValue& eov) override; + }; + + std::tuple...> appendages; + +public: +}; + +*/ + +} // namespace saw + +#include "async.tmpl.h" diff --git a/modules/async/async.tmpl.h b/modules/async/async.tmpl.h new file mode 100644 index 0000000..9569f60 --- /dev/null +++ b/modules/async/async.tmpl.h @@ -0,0 +1,767 @@ +#pragma once + +#include +#include + +#include +// Template inlining + +namespace saw { + +template conveyor_result execLater(Func &&func) { + conveyor conveyor{fix_void{}}; + return conveyor.then(std::move(func)); +} + +template +conveyor::conveyor(fix_void value) : conveyor_base(nullptr) { + // Is there any way to do this? + // @todo new conveyor_base constructor for Immediate values + + own>> immediate = + heap>>(std::move(value)); + + if (!immediate) { + return; + } + + node_ = std::move(immediate); +} + +template +conveyor::conveyor(error &&err) : conveyor_base(nullptr) { + own>> immediate = + heap>>(std::move(err)); + + if (!immediate) { + return; + } + + node_ = std::move(immediate); +} + +template +conveyor::conveyor(own node_p) + : conveyor_base{std::move(node_p)} {} + +template +template +conveyor_result conveyor::then(Func &&func, + ErrorFunc &&error_func) { + own conversion_node = + heap>, fix_void, + Func, ErrorFunc>>( + std::move(node_), std::move(func), std::move(error_func)); + + return conveyor>>::to_conveyor( + std::move(conversion_node)); +} + +template conveyor conveyor::buffer(size_t size) { + SAW_ASSERT(node_) { return conveyor{own{nullptr}}; } + conveyor_storage *storage = node_->next_storage(); + SAW_ASSERT(storage) { return conveyor{own{nullptr}}; } + + own>> storage_node = + heap>>(std::move(node_), size); + + conveyor_storage *storage_ptr = + static_cast(storage_node.get()); + + storage->set_parent(storage_ptr); + return conveyor{std::move(storage_node)}; +} + +template +template +conveyor conveyor::attach(Args &&...args) { + own> attach_node = + heap>(std::move(node_), + std::move(args...)); + return conveyor{std::move(attach_node)}; +} + +template +std::pair, merge_conveyor> conveyor::merge() { + our> data = + share>(); + + own> merge_node = heap>(data); + + SAW_ASSERT(node_) { + return std::make_pair(conveyor{own{nullptr}}, + merge_conveyor{}); + } + conveyor_storage *storage = node_->next_storage(); + SAW_ASSERT(storage) { + return std::make_pair(conveyor{own{nullptr}}, + merge_conveyor{}); + } + + data->attach(conveyor::to_conveyor(std::move(node_))); + + merge_conveyor node_ref{data}; + + return std::make_pair(conveyor{std::move(merge_node)}, + std::move(node_ref)); +} + +template <> +template +conveyor_sink conveyor::sink(ErrorFunc &&error_func) { + conveyor_storage *storage = node_->next_storage(); + SAW_ASSERT(storage) { return conveyor_sink{}; } + + own sink_node = + heap(std::move(node_)); + conveyor_storage *storage_ptr = + static_cast(sink_node.get()); + + storage->set_parent(storage_ptr); + + return conveyor_sink{std::move(sink_node)}; +} + +void detach_conveyor(conveyor &&conveyor); + +template +template +void conveyor::detach(ErrorFunc &&func) { + detach_conveyor(std::move(then([](T &&) {}, std::move(func)))); +} + +template <> +template +void conveyor::detach(ErrorFunc &&func) { + detach_conveyor(std::move(then([]() {}, std::move(func)))); +} + +template +conveyor conveyor::to_conveyor(own node) { + return conveyor{std::move(node)}; +} + +template +own conveyor::from_conveyor(conveyor conveyor) { + return std::move(conveyor.node_); +} + +template error_or> conveyor::take() { + SAW_ASSERT(node_) { + return error_or>{ + make_error("conveyor in invalid state")}; + } + conveyor_storage *storage = node_->next_storage(); + if (storage) { + if (storage->queued() > 0) { + error_or> result; + node_->get_result(result); + return result; + } else { + return error_or>{ + make_error("conveyor buffer has no elements")}; + } + } else { + return error_or>{ + make_error("conveyor node has no child storage")}; + } +} + +template conveyor_and_feeder new_conveyor_and_feeder() { + own>> feeder = + heap>>(); + own>> node = + heap>>(); + + feeder->set_feedee(node.get()); + node->set_feeder(feeder.get()); + + return conveyor_and_feeder{std::move(feeder), + conveyor::to_conveyor(std::move(node))}; +} + +// QueueBuffer +template void queue_buffer_conveyor_node::fire() { + if (child_mixin_.child) { + if (!storage_.empty()) { + if (storage_.front().is_error()) { + if (storage_.front().get_error().is_critical()) { + child_mixin_.child = nullptr; + } + } + } + } + + bool has_space_before_fire = space() > 0; + + if (parent_) { + parent_->child_has_fired(); + if (!storage_.empty() && parent_->space() > 0) { + arm_later(); + } + } + + if (!child_mixin_.child) { + while (!storage_.empty()) { + storage_.pop(); + } + return; + } + + conveyor_storage *ch_storage = child_mixin_.child->next_storage(); + if (ch_storage && !has_space_before_fire) { + ch_storage->parent_has_fired(); + } +} + +template +void queue_buffer_conveyor_node::get_result(error_or_value &eov) noexcept { + error_or &err_or_val = eov.as(); + err_or_val = std::move(storage_.front()); + storage_.pop(); +} + +template size_t queue_buffer_conveyor_node::space() const { + return max_store_ - storage_.size(); +} + +template size_t queue_buffer_conveyor_node::queued() const { + return storage_.size(); +} + +template void queue_buffer_conveyor_node::child_has_fired() { + if (child_mixin_.child && storage_.size() < max_store_) { + error_or eov; + child_mixin_.child->get_result(eov); + + if (eov.is_error()) { + if (eov.get_error().is_critical()) { + } + } + + storage_.push(std::move(eov)); + if (!is_armed()) { + arm_later(); + } + } +} + +template void queue_buffer_conveyor_node::parent_has_fired() { + SAW_ASSERT(parent_) { return; } + + if (parent_->space() == 0) { + return; + } + + if (queued() > 0) { + arm_later(); + } +} + +template +immediate_conveyor_node::immediate_conveyor_node(fix_void &&val) + : value_{std::move(val)}, retrieved_{0} {} + +template +immediate_conveyor_node::immediate_conveyor_node(error &&error) + : value_{std::move(error)}, retrieved_{0} {} + +template size_t immediate_conveyor_node::space() const { + return 0; +} + +template size_t immediate_conveyor_node::queued() const { + return retrieved_ > 1 ? 0 : 1; +} + +template void immediate_conveyor_node::child_has_fired() { + // Impossible case + assert(false); +} + +template void immediate_conveyor_node::parent_has_fired() { + SAW_ASSERT(parent_) { return; } + assert(parent_->space() > 0); + + if (queued() > 0) { + arm_next(); + } +} + +template void immediate_conveyor_node::fire() { + + if (parent_) { + parent_->child_has_fired(); + if (queued() > 0 && parent_->space() > 0) { + arm_last(); + } + } +} + +template +merge_conveyor::merge_conveyor(lent> d) + : data_{std::move(d)} {} + +template merge_conveyor::~merge_conveyor() {} + +template void merge_conveyor::attach(conveyor conveyor) { + auto sp = data_.lock(); + SAW_ASSERT(sp) { return; } + + sp->attach(std::move(conveyor)); +} + +template +merge_conveyor_node::merge_conveyor_node(our> d) + : data_{d} { + SAW_ASSERT(data_) { return; } + + data_->merger = this; +} + +template merge_conveyor_node::~merge_conveyor_node() {} + +template +error_or> +merge_conveyor_node::swap_child(own &&swapee_) noexcept { + (void)swapee_; + return make_error( + "merge_conveyor_node::appendage should block calls to this class"); +} + +template +void merge_conveyor_node::get_result(error_or_value &eov) noexcept { + error_or> &err_or_val = eov.as>(); + + SAW_ASSERT(data_) { return; } + + /// @todo search appendages for result + + auto &appendages = data_->appendages; + next_appendage_ = std::min(appendages.size(), next_appendage_); + + for (size_t i = next_appendage_; i < appendages.size(); ++i) { + if (appendages[i]->queued() > 0) { + err_or_val = std::move(appendages[i]->error_or_value_.value()); + appendages[i]->error_or_value_ = std::nullopt; + next_appendage_ = i + 1; + return; + } + } + for (size_t i = 0; i < next_appendage_; ++i) { + if (appendages[i]->queued() > 0) { + err_or_val = std::move(appendages[i]->error_or_value_.value()); + appendages[i]->error_or_value_ = std::nullopt; + next_appendage_ = i + 1; + return; + } + } + + err_or_val = make_error("No value in Merge appendages"); +} + +template void merge_conveyor_node::fire() { + SAW_ASSERT(queued() > 0) { return; } + + if (parent_) { + parent_->child_has_fired(); + + if (queued() > 0 && parent_->space() > 0) { + arm_later(); + } + } +} + +template size_t merge_conveyor_node::space() const { return 0; } + +template size_t merge_conveyor_node::queued() const { + SAW_ASSERT(data_) { return 0; } + + size_t queue_count = 0; + + for (auto &iter : data_->appendages) { + queue_count += iter->queued(); + } + + return queue_count; +} + +template void merge_conveyor_node::child_has_fired() { + /// This can never happen + assert(false); +} + +template void merge_conveyor_node::parent_has_fired() { + SAW_ASSERT(parent_) { return; } + if (queued() > 0) { + if (parent_->space() > 0) { + arm_later(); + } + } +} + +/** + * merge_conveyor_node::Apendage + */ + +template +error_or> +merge_conveyor_node::appendage::swap_child(own &&swapee_) { + own old_child = std::move(child); + + child = std::move(swapee_); + + // This case should never happen + SAW_ASSERT(old_child) { return make_error("No child exists"); } + + return old_child; +} + +template +void merge_conveyor_node::appendage::get_result(error_or_value &eov) { + error_or> &err_or_val = eov.as>(); + + SAW_ASSERT(queued() > 0) { + err_or_val = + make_error("No element queued in Merge appendage Node"); + return; + } + + err_or_val = std::move(error_or_value_.value()); + error_or_value_ = std::nullopt; +} + +template size_t merge_conveyor_node::appendage::space() const { + SAW_ASSERT(merger) { return 0; } + + if (error_or_value_.has_value()) { + return 0; + } + + return 1; +} + +template size_t merge_conveyor_node::appendage::queued() const { + SAW_ASSERT(merger) { return 0; } + + if (error_or_value_.has_value()) { + return 1; + } + + return 0; +} + +/// @todo delete this function. Replaced by the regular get_result +template +void merge_conveyor_node::appendage::get_appendage_result( + error_or_value &eov) { + error_or> &err_or_val = eov.as>(); + + SAW_ASSERT(queued() > 0) { + err_or_val = + make_error("No element queued in Merge appendage Node"); + return; + } + + err_or_val = std::move(error_or_value_.value()); + error_or_value_ = std::nullopt; +} + +template +void merge_conveyor_node::appendage::child_has_fired() { + SAW_ASSERT(!error_or_value_.has_value()) { return; } + error_or> eov; + child->get_result(eov); + + error_or_value_ = std::move(eov); + + if (!merger->is_armed()) { + merger->arm_later(); + } +} + +template +void merge_conveyor_node::appendage::parent_has_fired() { + conveyor_storage *child_storage = child->next_storage(); + if (child_storage) { + child_storage->parent_has_fired(); + } +} + +template +void merge_conveyor_node::appendage::set_parent(conveyor_storage *par) { + SAW_ASSERT(merger) { return; } + + SAW_ASSERT(child) { return; } + + parent_ = par; +} + +template +void merge_conveyor_node_data::attach(conveyor conv) { + auto nas = conveyor::from_conveyor(std::move(conv)); + SAW_ASSERT(nas) { return; } + conveyor_storage *storage = nas->next_storage(); + SAW_ASSERT(storage) { return; } + + auto merge_node_appendage = + heap::appendage>(std::move(nas), + *merger); + auto merge_node_appendage_ptr = merge_node_appendage.get(); + + storage->set_parent(merge_node_appendage.get()); + + SAW_ASSERT(merger) { return; } + + conveyor_storage *mrg_storage = merger->next_storage(); + SAW_ASSERT(mrg_storage) { return; } + + merge_node_appendage->set_parent(mrg_storage); + + appendages.push_back(std::move(merge_node_appendage)); + + /// @todo return this. necessary? maybe for the weird linking setup + /// maybe not + // return merge_node_appendage_ptr; +} + +template +void merge_conveyor_node_data::governing_node_destroyed() { + appendages.clear(); + merger = nullptr; +} + +template adapt_conveyor_feeder::~adapt_conveyor_feeder() { + if (feedee_) { + feedee_->set_feeder(nullptr); + feedee_ = nullptr; + } +} + +template +void adapt_conveyor_feeder::set_feedee(adapt_conveyor_node *feedee_p) { + feedee_ = feedee_p; +} + +template void adapt_conveyor_feeder::feed(T &&value) { + if (feedee_) { + feedee_->feed(std::move(value)); + } +} + +template void adapt_conveyor_feeder::fail(error &&error) { + if (feedee_) { + feedee_->fail(std::move(error)); + } +} + +template size_t adapt_conveyor_feeder::queued() const { + if (feedee_) { + return feedee_->queued(); + } + return 0; +} + +template size_t adapt_conveyor_feeder::space() const { + if (feedee_) { + return feedee_->space(); + } + return 0; +} + +template +error adapt_conveyor_feeder::swap(conveyor &&conv) noexcept { + SAW_ASSERT(feedee_) { return make_error("No feedee connected"); } + + auto node = conveyor::from_conveyor(std::move(conv)); + + feedee_->swap_child(std::move(node)); + + return no_error(); +} + +template +adapt_conveyor_node::adapt_conveyor_node() : conveyor_event_storage{} {} + +template adapt_conveyor_node::~adapt_conveyor_node() { + if (feeder_) { + feeder_->set_feedee(nullptr); + feeder_ = nullptr; + } +} + +template +error_or> +adapt_conveyor_node::swap_child(own &&swapee) noexcept { + // This should return the owning pointer of this instance + auto myself_err = parent_node_.swap_child_of_parent(std::move(swapee)); + + if (myself_err.is_error()) { + return myself_err; + } + + auto &myself = myself_err.get_value(); + + assert(myself.get() == this); + + return myself_err; +} + +template +conveyor_storage *adapt_conveyor_node::next_storage() noexcept { + return static_cast(this); +} + +template +void adapt_conveyor_node::notify_parent_attached( + conveyor_node &par) noexcept { + parent_node_.change_parent(&par); +} + +template +void adapt_conveyor_node::set_feeder(adapt_conveyor_feeder *feeder_p) { + feeder_ = feeder_p; +} + +template void adapt_conveyor_node::feed(T &&value) { + storage_.push(std::move(value)); + arm_next(); +} + +template void adapt_conveyor_node::fail(error &&error) { + storage_.push(std::move(error)); + arm_next(); +} + +template size_t adapt_conveyor_node::queued() const { + return storage_.size(); +} + +template size_t adapt_conveyor_node::space() const { + return std::numeric_limits::max() - storage_.size(); +} + +template +void adapt_conveyor_node::get_result(error_or_value &err_or_val) { + if (!storage_.empty()) { + err_or_val.as() = std::move(storage_.front()); + storage_.pop(); + } else { + err_or_val.as() = make_error( + "Signal for retrieval of storage sent even though no " + "data is present"); + } +} + +template void adapt_conveyor_node::child_has_fired() { + // Adapt node has no children + assert(false); +} + +template void adapt_conveyor_node::parent_has_fired() { + SAW_ASSERT(parent_) { return; } + + if (parent_->space() == 0) { + return; + } +} + +template void adapt_conveyor_node::fire() { + if (parent_) { + parent_->child_has_fired(); + + if (storage_.size() > 0) { + arm_later(); + } + } +} + +template one_time_conveyor_feeder::~one_time_conveyor_feeder() { + if (feedee_) { + feedee_->set_feeder(nullptr); + feedee_ = nullptr; + } +} + +template +void one_time_conveyor_feeder::set_feedee( + one_time_conveyor_node *feedee_p) { + feedee_ = feedee_p; +} + +template void one_time_conveyor_feeder::feed(T &&value) { + if (feedee_) { + feedee_->feed(std::move(value)); + } +} + +template void one_time_conveyor_feeder::fail(error &&error) { + if (feedee_) { + feedee_->fail(std::move(error)); + } +} + +template size_t one_time_conveyor_feeder::queued() const { + if (feedee_) { + return feedee_->queued(); + } + return 0; +} + +template size_t one_time_conveyor_feeder::space() const { + if (feedee_) { + return feedee_->space(); + } + return 0; +} + +template one_time_conveyor_node::~one_time_conveyor_node() { + if (feeder_) { + feeder_->set_feedee(nullptr); + feeder_ = nullptr; + } +} + +template +void one_time_conveyor_node::set_feeder( + one_time_conveyor_feeder *feeder_p) { + feeder_ = feeder_p; +} + +template void one_time_conveyor_node::feed(T &&value) { + storage_ = std::move(value); + arm_next(); +} + +template void one_time_conveyor_node::fail(error &&error) { + storage_ = std::move(error); + arm_next(); +} + +template size_t one_time_conveyor_node::queued() const { + return storage_.has_value() ? 1 : 0; +} + +template size_t one_time_conveyor_node::space() const { + return passed_ ? 0 : 1; +} + +template +void one_time_conveyor_node::get_result(error_or_value &err_or_val) { + if (storage_.has_value()) { + err_or_val.as() = std::move(storage_.value()); + storage_ = std::nullopt; + } else { + err_or_val.as() = make_error( + "Signal for retrieval of storage sent even though no " + "data is present"); + } +} + +template void one_time_conveyor_node::fire() { + if (parent_) { + parent_->child_has_fired(); + } +} + +} // namespace saw diff --git a/modules/codec-json/.nix/derivation.nix b/modules/codec-json/.nix/derivation.nix new file mode 100644 index 0000000..3738e43 --- /dev/null +++ b/modules/codec-json/.nix/derivation.nix @@ -0,0 +1,30 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-codec-json"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/codec-json/SConscript b/modules/codec-json/SConscript new file mode 100644 index 0000000..772ac0b --- /dev/null +++ b/modules/codec-json/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +codec_json_env = env.Clone(); + +codec_json_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +codec_json_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += codec_json_env.sources; +env.headers += codec_json_env.headers; + +## Shared lib +objects_shared = [] +codec_json_env.add_source_files(objects_shared, codec_json_env.sources, shared=True); +codec_json_env.library_shared = codec_json_env.SharedLibrary('#build/forstio-codec-json', [objects_shared]); + +## Static lib +objects_static = [] +codec_json_env.add_source_files(objects_static, codec_json_env.sources, shared=False); +codec_json_env.library_static = codec_json_env.StaticLibrary('#build/forstio-codec-json', [objects_static]); + +# Set Alias +env.Alias('library_codec_json', [codec_json_env.library_shared, codec_json_env.library_static]); + +env.targets += ['library_codec_json']; + +# Install +env.Install('$prefix/lib/', [codec_json_env.library_shared, codec_json_env.library_static]); +env.Install('$prefix/include/forstio/codec/json/', [codec_json_env.headers]); diff --git a/modules/codec-json/SConstruct b/modules/codec-json/SConstruct new file mode 100644 index 0000000..edd5f57 --- /dev/null +++ b/modules/codec-json/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-codec']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/codec-json/json.h b/modules/codec-json/json.h new file mode 100644 index 0000000..56cadc0 --- /dev/null +++ b/modules/codec-json/json.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include +#include + +#include + +namespace saw { +namespace encode { +struct Json {}; +} + +template +class data { +private: + ring_buffer buffer_; +public: + data():buffer_{}{} + + data(std::size_t ring_size_):buffer_{ring_size_}{} + + data(ring_buffer buff_): + buffer_{std::move(buff_)} + {} + + data(const std::string_view& view__): + buffer_{view__.size()} + { + auto ptr = reinterpret_cast(view__.data()); + if(!ptr){ + return; + } + buffer_.push(*ptr, view__.size()); + } + + buffer& get_buffer(){ + return buffer_; + } + + const buffer& get_buffer() const { + return buffer_; + } + + error push(uint8_t val){ + return buffer_.push(val); + } + + std::size_t get_size() const { + return buffer_.read_composite_length(); + } + + uint8_t& at(std::size_t i){ + return buffer_.read(i); + } + + const uint8_t& at(std::size_t i) const { + return buffer_.read(i); + } +}; +} + +#include "json.tmpl.h" + +namespace saw { + +/** + * Codec class for json + */ +template +class codec { +public: + struct config { + size_t depth = 16; + size_t length = 1024; + }; +private: + config cfg_; +public: + /** + * Default constructor + */ + codec(){} + + /** + * Constructor + */ + codec(config cfg__):cfg_{std::move(cfg__)}{} + + SAW_FORBID_COPY(codec); + SAW_DEFAULT_MOVE(codec); + + template + error_or encode(const data& from_encode, data& to_encode){ + // To Be encoded + buffer_view buff_v{to_encode.get_buffer()}; + auto eov = impl::json_encode::encode(from_encode, buff_v); + if(eov.is_error()){ + return std::move(eov.get_error()); + } + to_encode.get_buffer().write_advance(buff_v.write_offset()); + + return void_t{}; + } + + template + error_or decode(data& from_decode, data& to_decode){ + buffer_view buff_v{from_decode.get_buffer()}; + + auto eov = impl::json_decode::decode(buff_v, to_decode); + if(eov.is_error()){ + return std::move(eov.get_error()); + } + from_decode.get_buffer().read_advance(buff_v.read_offset()); + + return void_t {}; + } +}; +} + diff --git a/modules/codec-json/json.tmpl.h b/modules/codec-json/json.tmpl.h new file mode 100644 index 0000000..4b3a1a2 --- /dev/null +++ b/modules/codec-json/json.tmpl.h @@ -0,0 +1,734 @@ +#pragma once + +#include +#include + +#include + +namespace saw { +namespace impl { +template +class json_encode { + static_assert(always_false, "This schema type is not being handled by the json encoding."); +}; + +template +struct json_encode, RootSchema, FromEncode> { + static error_or encode(const data, FromEncode>& from, buffer& to) { + auto val = from.get(); + std::array data; + auto tc_result = std::to_chars(reinterpret_cast(data.data()), reinterpret_cast(data.data())+data.size(), val); + + if(tc_result.ec != std::errc{}){ + return make_error(); + } + + size_t bytes_written = 0; + for(char* ptr = reinterpret_cast(data.data()); ptr != tc_result.ptr; ++ptr){ + ++bytes_written; + } + + auto& buff = to; + error err = buff.write_require_length(bytes_written); + if(!err.template is_type()){ + return std::move(err); + } + + for(char* ptr = reinterpret_cast(data.data()); ptr != tc_result.ptr; ++ptr){ + uint8_t* un_ptr = reinterpret_cast(ptr); + buff.push(un_ptr[0]); + } + + return void_t{}; + } +}; + +template +struct json_encode { + static error_or encode(const data& from, buffer& to) { + { + auto err = to.push('"'); + if(!err.template is_type()){ + return err; + } + } + for(std::size_t i = 0; i < from.size(); ++i){ + auto err = to.push(from.get_at(i)); + if(!err.template is_type()){ + return err; + } + } + { + auto err = to.push('"'); + if(!err.template is_type()){ + return err; + } + } + + return void_t{}; + } +}; + +template +struct json_encode, RootSchema, FromEncode> { + template + static error_or encode_element(const data, FromEncode>& from, buffer& to){ + auto eov = json_encode::type, RootSchema, FromEncode>::encode(from.template get(), to); + + if(eov.is_error()){ + return eov; + } + + if constexpr ( (i+1) < sizeof...(T)){ + { + auto eov_ele = to.push(','); + if(!eov_ele.template is_type()){ + return eov_ele; + } + } + { + auto eov_ele = encode_element(from, to); + if(eov_ele.is_error()){ + return eov_ele; + } + } + + + } + return void_t{}; + } + + static error_or encode(const data, FromEncode>& from, buffer& to) { + { + auto err = to.push('['); + if(!err.template is_type()){ + return err; + } + } + if constexpr ( sizeof...(T) > 0 ){ + auto eov = encode_element<0>(from, to); + if(eov.is_error()){ + return eov; + } + } + { + auto err = to.push(']'); + if(!err.template is_type()){ + return err; + } + } + + return void_t{}; + } +}; + +template +struct json_encode, RootSchema, FromEncode> { + template + static error_or encode_level(const data, FromEncode>& from, buffer& to, std::array& index){ + if constexpr (Level == D){ + auto eov = json_encode::encode(from.at(index), to); + if(eov.is_error()){ + return eov; + } + } else { + { + auto err = to.push('['); + if(!err.template is_type()){ + return err; + } + } + for(std::size_t i = 0; i < from.get_dim_size(Level); ++i){ + if( i > 0 ){ + auto err = to.push(','); + if(!err.template is_type()){ + return err; + } + } + { + index[Level] = i; + auto eov = encode_level(from, to, index); + if(eov.is_error()){ + return eov; + } + } + } + { + auto err = to.push(']'); + if(!err.template is_type()){ + return err; + } + } + } + return void_t{}; + } + + static error_or encode(const data, FromEncode>& from, buffer& to) { + std::array index; + return encode_level<0>(from, to, index); + } +}; + +template +struct json_encode...>, RootSchema, FromEncode> { + + template + static error_or encode_element(const data...>, FromEncode>& from, buffer& to){ + // Encode the name + { + std::string_view view = parameter_key_pack_type::literal.view(); + error err = to.push('"'); + if(!err.template is_type()){ + return err; + } + err = to.push(*reinterpret_cast(view.data()), view.size()); + if(!err.template is_type()){ + return err; + } + err = to.push('"'); + if(!err.template is_type()){ + return err; + } + } + // Add the separator + { + auto eov_ele = to.push(':'); + if(!eov_ele.template is_type()){ + return eov_ele; + } + } + + // Encode the value + auto eov = json_encode::type, RootSchema, FromEncode>::encode(from.template get::literal>(), to); + + // Go to the next element + if constexpr ( (i+1) < sizeof...(T)){ + { + auto eov_ele = to.push(','); + if(!eov_ele.template is_type()){ + return eov_ele; + } + } + { + auto eov_ele = encode_element(from, to); + if(eov_ele.is_error()){ + return eov_ele; + } + } + + + } + + return void_t{}; + } + static error_or encode(const data...>, FromEncode>& from, buffer& to) { + { + auto err = to.push('{'); + if(!err.template is_type()){ + return err; + } + } + if constexpr ( sizeof...(T) > 0 ){ + auto eov = encode_element<0>(from, to); + if(eov.is_error()){ + return eov; + } + } + { + auto err = to.push('}'); + if(!err.template is_type()){ + return err; + } + } + + return void_t{}; + } +}; + +template +struct json_encode...>, RootSchema, FromEncode> { + + template + static error_or encode_element(const data...>, FromEncode>& from, buffer& to){ + if(from.template holds_alternative::literal>()){ + // Encode the name + { + std::string_view view = parameter_key_pack_type::literal.view(); + error err = to.push('"'); + if(!err.template is_type()){ + return err; + } + err = to.push(*reinterpret_cast(view.data()), view.size()); + if(!err.template is_type()){ + return err; + } + err = to.push('"'); + if(!err.template is_type()){ + return err; + } + } + // Add the separator + { + auto eov_ele = to.push(':'); + if(!eov_ele.template is_type()){ + return eov_ele; + } + } + + // Encode the value + auto eov = json_encode::type, RootSchema, FromEncode>::encode(from.template get::literal>(), to); + } + // Go to the next element + if constexpr ( (i+1) < sizeof...(T)){ + { + auto eov_ele = encode_element(from, to); + if(eov_ele.is_error()){ + return eov_ele; + } + } + + + } + + return void_t{}; + } + static error_or encode(const data...>, FromEncode>& from, buffer& to) { + { + auto err = to.push('{'); + if(!err.template is_type()){ + return err; + } + } + if constexpr ( sizeof...(T) > 0 ){ + auto eov = encode_element<0>(from, to); + if(eov.is_error()){ + return eov; + } + } + { + auto err = to.push('}'); + if(!err.template is_type()){ + return err; + } + } + + return void_t{}; + } +}; + +struct json_helper { + static bool is_whitespace(int8_t ch) { + return ch == '\t' || ch == ' ' || ch == '\r' || ch == '\n'; + } + + static void skip_whitespace(buffer_view& buff) { + while(buff.read_composite_length() > 0 && json_helper::is_whitespace(buff.read())) { + buff.read_advance(1); + } + } +}; + +template +struct json_decode; + +template +struct json_decode, RootSchema, ToDecode> { + static error_or decode(buffer_view& buff, data, ToDecode>& to) { + assert( + ( buff.read() >= '0' && buff.read() <= '9') + || ( buff.read() == '+' || buff.read() == '-') + ); + + std::size_t offset = 0; + + if (buff.read() == '-'){ + ++offset; + } else if (buff.read() == '+'){ + return make_error(); + } + if (offset >= buff.read_composite_length()) { + return make_error(); + } + if (buff.read(offset) >= '1' && buff.read(offset) <= '9') { + ++offset; + + if(offset >= buff.read_composite_length()) { + return make_error(); + } + + while(1){ + if (buff.read(offset) >= '0' && buff.read(offset) <= '9') { + ++offset; + + if(offset >= buff.read_composite_length()) { + break; + } + continue; + } + break; + } + } else if (buff.read(offset) == '0' ) { + ++offset; + } else { + return make_error(); + } + + { + std::string_view num_view{reinterpret_cast(&buff.read()), offset}; + typename native_data_type>::type result; + auto fc_result = std::from_chars(num_view.data(), num_view.data() + num_view.size(), result); + if(fc_result.ec != std::errc{}){ + return make_error(); + } + + to.set(result); + } + buff.read_advance(offset); + + return void_t{}; + } +}; + +template +struct json_decode { + static error_or decode(buffer_view& buff, data& to){ + assert(buff.read() == '"'); + buff.read_advance(1); + + std::stringstream iss; + bool string_done = false; + while(!string_done){ + if(buff.read_composite_length() == 0){ + return make_error(); + } + + switch(buff.read()){ + case '\\':{ + buff.read_advance(1); + if(buff.read_composite_length() == 0){ + return make_error(); + } + switch(buff.read()){ + case '\\': + case '/': + case '"': + iss<< buff.read(); + break; + case 'b': + iss<<'\b'; + break; + case 'f': + iss<<'\f'; + break; + case 'n': + iss<<'\n'; + break; + case 'r': + iss<<'\r'; + break; + case 't': + iss<<'\t'; + break; + case 'u': { + buff.read_advance(1); + if(buff.read_composite_length() < 4){ + return make_error(); + } + iss<<'?'; + iss<<'?'; + iss<<'?'; + iss<<'?'; + + buff.read_advance(3); + } break; + } + } break; + case '"': + string_done = true; + break; + default:{ + // Avoids Control sequences + if(buff.read() >= ' ' && buff.read() <= '~'){ + iss<(); + } + } while( buff.read() < 0 ); + iss<<'?'; + } + break; + } + } + buff.read_advance(1); + } + + std::string raw = iss.str(); + to.set(std::move(raw)); + + return void_t{}; + } +}; + +template +struct json_decode...>, RootSchema, ToDecode> { + template + static error_or decode_field_search(buffer_view& buff, data...>, ToDecode>& to, std::array& fields, const data& search_name){ + if constexpr ( i < sizeof...(T)){ + using Type = typename parameter_pack_type::type; + constexpr static string_literal Literal = parameter_key_pack_type::literal; + if(search_name == Literal.view()){ + if(fields[i]){ + return make_error(); + } + fields[i] = true; + auto eov = json_decode::decode(buff, to.template get()); + if(eov.is_error()){ + return eov; + } + }else { + decode_field_search(buff, to, fields, search_name); + } + }else { + return make_error(); + } + return void_t{}; + } + + static error_or decode_fields(buffer_view& buff, data...>, ToDecode>& to, std::array& fields){ + for(;;){ + data name; + auto eov = json_decode::decode(buff, name); + if(eov.is_error()){ + return eov; + } + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + if(buff.read() != ':'){ + return make_error(); + } + buff.read_advance(1); + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + { + auto eov = decode_field_search<0>(buff, to, fields, name); + if(eov.is_error()){ + return eov; + } + } + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + if(buff.read() == ','){ + buff.read_advance(1); + }else if(buff.read() == '}'){ + // If not all fields are set, the dataset is incomplete + for(auto& iter : fields){ + if(!iter){ + return make_error(); + } + } + buff.read_advance(1); + return void_t{}; + }else{ + return make_error(); + } + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + } + return void_t{}; + } + + static error_or decode(buffer_view& buff, data...>, ToDecode>& to){ + std::array found_fields; + std::fill(found_fields.begin(), found_fields.end(), false); + + assert(buff.read() == '{'); + buff.read_advance(1); + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + + // Check if there are no elements present in the JSON Struct + if(buff.read() == '}'){ + if(sizeof...(T) > 0){ + return make_error(); + } + buff.read_advance(1); + return void_t{}; + } + + auto eov = decode_fields(buff, to, found_fields); + if(eov.is_error()){ + return eov; + } + + return void_t{}; + } +}; + +template +struct json_decode, RootSchema, ToDecode> { + template + static error_or decode_element(buffer_view& buff, data, ToDecode>& to){ + if constexpr (i < sizeof...(T)){ + if constexpr ( i > 0 ){ + if(buff.read() != ','){ + return make_error(); + } + buff.read_advance(1); + if(buff.read_composite_length() == 0){ + return make_error(); + } + } + using Type = typename parameter_pack_type::type; + + auto eov = json_decode::decode(buff, to.template get()); + if(eov.is_error()){ + return eov; + } + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + + eov = decode_element(buff, to); + if(eov.is_error()){ + return eov; + } + }else{ + if(buff.read() != ']'){ + return make_error(); + } + buff.read_advance(1); + } + return void_t{}; + } + + static error_or decode(buffer_view& buff, data, ToDecode>& to){ + assert(buff.read() == '['); + buff.read_advance(1); + + json_helper::skip_whitespace(buff); + if(buff.read_composite_length() == 0){ + return make_error(); + } + + auto eov = decode_element<0>(buff, to); + if(eov.is_error()){ + return eov; + } + + return void_t{}; + } +}; + +// The whole std::vector approach is hacky af. ToDo change it maybe? +template +struct json_decode, RootSchema, ToDecode> { + template + static error_or decode_flat_level(buffer_view& buff, std::vector>& to, std::array& index, std::array& dims, bool log_dim){ + if constexpr (Level == D) { + json_helper::skip_whitespace(buff); + try { + to.push_back({}); + }catch(std::exception& e){ + return make_error(); + } + auto eov = json_decode::decode(buff, to.back()); + if(eov.is_error()){ + return eov; + } + } else { + assert(buff.read() == '['); + buff.read_advance(1); + + json_helper::skip_whitespace(buff); + if ( buff.read_composite_length() == 0 ){ + return make_error(); + } + + index[Level] = 0; + for(;;){ + // We should have an element right now + auto eov = decode_flat_level(buff,to,index,dims, index[Level] == 0 && log_dim); + if(eov.is_error()){ + return eov; + } + json_helper::skip_whitespace(buff); + if ( buff.read_composite_length() == 0 ){ + return make_error(); + } + + ++index[Level]; + if(buff.read() == ','){ + buff.read_advance(1); + } else if(buff.read() == ']'){ + buff.read_advance(1); + break; + } else { + return make_error(); + } + json_helper::skip_whitespace(buff); + if ( buff.read_composite_length() == 0 ){ + return make_error(); + } + } + if(log_dim){ + dims[Level] = index[Level]; + }else if (dims[Level] != index[Level]){ + return make_error(); + } + } + return void_t{}; + } + + template + static error_or decode_unflat_level(std::vector>& flat, data, ToDecode>& to, std::array& index, std::size_t& flat_index) { + if constexpr ( Level == D ){ + auto& flat_data = flat.at(flat_index); + to.at(index) = std::move(flat_data); + ++flat_index; + }else { + const std::size_t dim_size = to.get_dim_size(Level); + for(index[Level] = 0; index[Level] < dim_size; ++index[Level]){ + + auto eov = decode_unflat_level(flat, to, index, flat_index); + if(eov.is_error()){ + return eov; + } + } + } + return void_t{}; + } + + static error_or decode(buffer_view& buff, data, ToDecode>& to){ + std::array index; + std::array dims; + std::fill(dims.begin(), dims.end(), 0); + std::vector> flat_array; + auto eov = decode_flat_level<0>(buff, flat_array, index, dims, true); + if(eov.is_error()){ + return eov; + } + + to = {dims}; + std::size_t flat_index = 0; + + return decode_unflat_level<0>(flat_array, to, index, flat_index); + } +}; +} +} diff --git a/modules/codec-minecraft/.nix/derivation.nix b/modules/codec-minecraft/.nix/derivation.nix new file mode 100644 index 0000000..7704d13 --- /dev/null +++ b/modules/codec-minecraft/.nix/derivation.nix @@ -0,0 +1,32 @@ +{ lib +, stdenvNoCC +, scons +, clang +, clang-tools +, version +, forstio +}: + +let + +in stdenvNoCC.mkDerivation { + pname = "forstio-codec-minecraft"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/codec-minecraft/SConscript b/modules/codec-minecraft/SConscript new file mode 100644 index 0000000..4d1deab --- /dev/null +++ b/modules/codec-minecraft/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +codec_minecraft_env = env.Clone(); + +codec_minecraft_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +codec_minecraft_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += codec_minecraft_env.sources; +env.headers += codec_minecraft_env.headers; + +## Shared lib +objects_shared = [] +codec_minecraft_env.add_source_files(objects_shared, codec_minecraft_env.sources, shared=True); +codec_minecraft_env.library_shared = codec_minecraft_env.SharedLibrary('#build/forstio-codec-minecraft', [objects_shared]); + +## Static lib +objects_static = [] +codec_minecraft_env.add_source_files(objects_static, codec_minecraft_env.sources, shared=False); +codec_minecraft_env.library_static = codec_minecraft_env.StaticLibrary('#build/forstio-codec-minecraft', [objects_static]); + +# Set Alias +env.Alias('library_codec_minecraft', [codec_minecraft_env.library_shared, codec_minecraft_env.library_static]); + +env.targets += ['library_codec_minecraft']; + +# Install +env.Install('$prefix/lib/', [codec_minecraft_env.library_shared, codec_minecraft_env.library_static]); +env.Install('$prefix/include/forstio/codec/minecraft/', [codec_minecraft_env.headers]); diff --git a/modules/codec-minecraft/SConstruct b/modules/codec-minecraft/SConstruct new file mode 100644 index 0000000..edd5f57 --- /dev/null +++ b/modules/codec-minecraft/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-codec']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/codec-minecraft/minecraft.h b/modules/codec-minecraft/minecraft.h new file mode 100644 index 0000000..670b019 --- /dev/null +++ b/modules/codec-minecraft/minecraft.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include + +namespace saw { +namespace encode { +struct Minecraft {}; +struct VarIntTransport {}; +} + +template +class data { +private: + ring_buffer buffer_; +public: + data() = default; + data(std::size_t size): + buffer_{size} + {} + + buffer& get_buffer(){ + return buffer_; + } +}; + +namespace mc { +namespace impl { +union minecraft_signed_four_conversion { + int32_t s; + uint32_t u; +}; + +union minecraft_signed_eight_conversion { + int64_t s; + uint64_t u; +}; + +template +class minecraft_encode { + static_assert(always_false, "This schema type is not being handled by the Minecraft encoding."); +}; + +template +class minecraft_encode{ + static error_or encode(const data& from, buffer& to){ + uint8_t encode_index = 0; + minecraft_signed_four_conversion value; + value.s = from.get(); + + /** + * VarInt max length is 5 bytes + */ + std::array encode_data; + + do { + uint8_t step = static_cast(value.u & 0x7F); + value.u = value.u >> 7; + if(value.u != 0){ + step |= 0x80; + } + encode_data[encode_index] = step; + ++encode_index; + }while(value.u != 0); + + auto err = buffer.push(encode_data[0], encode_index); + if (!err.template is_type()) { + return err; + } + + return no_error(); + } +}; + +template +class minecraft_decode { + static_assert(always_false, "This schema type is not being handled by the Minecraft encoding."); +}; + +template +class minecraft_decode{ + static error_or decode(buffer& from, data& to){ + uint8_t num_reads = 0; + + minecraft_signed_four_conversion value; + value.u = 0; + + uint8_t read{}; + do { + auto err = from.pop(read); + if( !err.template is_type() ){ + return err; + } + value.u |= ((read & 0x7F) << (7*num_reads)); + ++num_reads; + if(num_reads > 5){ + return make_error(); + } + } while( (read & 0x80) != 0); + + to.set(value.s); + + return no_error(); + } +}; + +} +} + + + +namespace mc { + +} +} diff --git a/modules/codec-minecraft/var_int_data.cpp b/modules/codec-minecraft/var_int_data.cpp new file mode 100644 index 0000000..59e4317 --- /dev/null +++ b/modules/codec-minecraft/var_int_data.cpp @@ -0,0 +1,36 @@ +#include "var_int_data.h" + +namespace saw { +data::data(): + data_{} +{} + +data::data(int32_t data): + data_{data} +{} + +int32_t data::get() const { + return data_; +} + +void data::set(int32_t data) { + data_ = data; +} + +data::data(): + data_{} +{} + +data::data(int64_t data): + data_{data} +{} + +int64_t data::get() const { + return data_; +} + +void data::set(int64_t data) { + data_ = data; +} + +} diff --git a/modules/codec-minecraft/var_int_data.h b/modules/codec-minecraft/var_int_data.h new file mode 100644 index 0000000..807c930 --- /dev/null +++ b/modules/codec-minecraft/var_int_data.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace saw { +template<> +class data { +private: + int32_t data_; +public: + data(); + data(int32_t data); + + int32_t get() const; + void set(int32_t); +}; + +template<> +class data { +private: + int64_t data_; +public: + data(); + data(int64_t data); + + int64_t get() const; + void set(int64_t); +}; + + +} diff --git a/modules/codec-netcdf/.nix/derivation.nix b/modules/codec-netcdf/.nix/derivation.nix new file mode 100644 index 0000000..770942e --- /dev/null +++ b/modules/codec-netcdf/.nix/derivation.nix @@ -0,0 +1,33 @@ +{ lib +, stdenv +, scons +, clang +, clang-tools +, version +, forstio +, netcdf +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-codec-netcdf"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + netcdf + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/codec-netcdf/SConscript b/modules/codec-netcdf/SConscript new file mode 100644 index 0000000..a469f77 --- /dev/null +++ b/modules/codec-netcdf/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +codec_netcdf_env = env.Clone(); + +codec_netcdf_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +codec_netcdf_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += codec_netcdf_env.sources; +env.headers += codec_netcdf_env.headers; + +## Shared lib +objects_shared = [] +codec_netcdf_env.add_source_files(objects_shared, codec_netcdf_env.sources, shared=True); +codec_netcdf_env.library_shared = codec_netcdf_env.SharedLibrary('#build/forstio-codec-netcdf', [objects_shared]); + +## Static lib +objects_static = [] +codec_netcdf_env.add_source_files(objects_static, codec_netcdf_env.sources, shared=False); +codec_netcdf_env.library_static = codec_netcdf_env.StaticLibrary('#build/forstio-codec-netcdf', [objects_static]); + +# Set Alias +env.Alias('library_codec_netcdf', [codec_netcdf_env.library_shared, codec_netcdf_env.library_static]); + +env.targets += ['library_codec_netcdf']; + +# Install +env.Install('$prefix/lib/', [codec_netcdf_env.library_shared, codec_netcdf_env.library_static]); +env.Install('$prefix/include/forstio/codec/netcdf/', [codec_netcdf_env.headers]); diff --git a/modules/codec-netcdf/SConstruct b/modules/codec-netcdf/SConstruct new file mode 100644 index 0000000..edd5f57 --- /dev/null +++ b/modules/codec-netcdf/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-codec']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/codec-netcdf/netcdf.h b/modules/codec-netcdf/netcdf.h new file mode 100644 index 0000000..cc9aedb --- /dev/null +++ b/modules/codec-netcdf/netcdf.h @@ -0,0 +1,151 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace saw { +namespace encode { +/** + * Template type hint + */ +struct Netcdf {}; +} + +/** + * Class representing the files system netcdf file + */ +template +class data { +private: + std::vector buff_; +public: + data() = default; + + data(std::vector buff): + buff_{std::move(buff)} + {} + + template + data(const std::array& arr): + buff_{arr.begin(), arr.end()} + {} + + std::vector& get_data() { + return buff_; + } + + const std::vector& get_data() const { + return buff_; + } +}; + +template +class codec{ + static_assert(always_false, "NetCDF only supports Structs as a root object"); +}; + +} +#include "netcdf.tmpl.h" +namespace saw { + +template +class codec...>, encode::Netcdf> { +private: + using Schema = schema::Struct...>; +public: + SAW_FORBID_COPY(codec); + SAW_DEFAULT_MOVE(codec); + + /** + * Default constructor + */ + codec() = default; + + /** + * Encoder function + */ + template + error_or encode(const data& from, data& to) { + int rc{}; + int ncid{}; + + rc = nc_create_mem("forstio_internal_write_memory", 0,/*NC_NETCDF4, */0, &ncid); + if(rc != NC_NOERR){ + return make_error(); + } + + { + auto eov = impl::netcdf_encode::encode_meta(from, ncid); + if(eov.is_error()){ + nc_close(ncid); + return eov; + } + } + + rc = nc_enddef(ncid); + if(rc != NC_NOERR){ + nc_close(ncid); + return make_error(); + } + + { + auto eov = impl::netcdf_encode::encode(from, ncid); + if(eov.is_error()){ + nc_close(ncid); + return eov; + } + } + + NC_memio nc_memio; + rc = nc_close_memio(ncid, &nc_memio); + if(rc != NC_NOERR){ + return make_error(); + } + + if(!nc_memio.memory || nc_memio.size == 0){ + return make_error(); + } + + try { + to.get_data().resize(nc_memio.size); + }catch(const std::exception& e){ + (void) e; + return make_error(); + } + + for(std::size_t i = 0; i < nc_memio.size; ++i){ + to.get_data().at(i) = static_cast(nc_memio.memory)[i]; + } + + free(nc_memio.memory); + nc_memio.memory = nullptr; + + return void_t{}; + } + + /** + * Decoder function + */ + template + error_or decode(data& from_decode, data& to_decode) { + int ncid{}; + int rc{}; + + rc = nc_open_mem("forstio_internal_read_memory", NC_NOWRITE, from_decode.get_data().size(), &from_decode.get_data()[0], &ncid); + if(rc != NC_NOERR){ + // Don't know how to get the error, so fail critically. + return make_error(); + } + + auto eov = impl::netcdf_decode::decode(to_decode, ncid); + + nc_close(ncid); + + return eov; + } +}; +} diff --git a/modules/codec-netcdf/netcdf.tmpl.h b/modules/codec-netcdf/netcdf.tmpl.h new file mode 100644 index 0000000..bf257e4 --- /dev/null +++ b/modules/codec-netcdf/netcdf.tmpl.h @@ -0,0 +1,348 @@ +#pragma once + +namespace saw { +namespace impl { +template +struct netcdf_primitive_id { + static_assert(always_false, "Not a primitive id"); +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_INT; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_UINT; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_INT64; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_UINT64; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_FLOAT; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_DOUBLE; +}; + +template +struct netcdf_is_group { + static constexpr bool value = false; +}; + +template +struct netcdf_is_group...>> { + static constexpr bool value = true; +}; + +template +struct netcdf_encode; + +template +struct netcdf_encode, ToEncode> { + using Schema = schema::Primitive; + + static error_or encode(const data& from, int ncid, int ncvarid){ + int rc{}; + + typename native_data_type::type val = from.get(); + + rc = nc_put_var(ncid, ncvarid, &val); + if(rc != NC_NOERR) { + return make_error(); + } + + return void_t{}; + } + + static error_or encode_meta(const data& from, int ncid, int ncvarid){ + (void) from; + (void) ncid; + (void) ncvarid; + return void_t{}; + } +}; + +template +struct netcdf_encode, ToEncode> { + using Schema = schema::Array; + + template + static error_or encode_level(data& from, int ncid, int ncvarid){ + + return make_error(); + } + + static error_or encode(data& from, int ncid, int ncvarid){ + + return make_error(); + } + + static error_or encode_meta(const data& from, int ncid, int ncvarid){ + + return make_error(); + } +}; + +template +struct netcdf_encode...>, ToEncode> { + using Schema = schema::Struct...>; + + template + static error_or encode_member(const data& from, int ncid){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + /** + * We have to ask for the internal id of the netcdf structure + */ + if constexpr (netcdf_is_group::value){ + return make_error(); + }else{ + int nc_varid{}; + int rc {}; + rc = nc_inq_varid(ncid, Literal.data.data(), &nc_varid); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_encode::encode(from.template get(), ncid, nc_varid); + if(eov.is_error()){ + return eov; + } + } + + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = encode_member(from, ncid); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } + + static error_or encode(const data& from, int ncid){ + if constexpr (sizeof...(T) > 0){ + auto eov = encode_member<0>(from, ncid); + return eov; + } + + return void_t{}; + } + + template + static error_or encode_meta_member(const data& from, int ncid){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + /** + * We have to ask for the internal id of the netcdf structure + */ + if constexpr (netcdf_is_group::value){ + return make_error(); + }else{ + int nc_varid{}; + int rc {}; + rc = nc_def_var(ncid, Literal.data.data(), netcdf_primitive_id::value, 0, nullptr, &nc_varid); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_encode::encode_meta(from.template get(), ncid, nc_varid); + if(eov.is_error()){ + return eov; + } + } + + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = encode_meta_member(from, ncid); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } + + static error_or encode_meta(const data& from, int ncid){ + if constexpr (sizeof...(T) > 0){ + auto eov = encode_meta_member<0>(from, ncid); + return eov; + } + + return void_t{}; + } +}; + +template +struct netcdf_decode; + +template +struct netcdf_decode, ToDecode> { + using Schema = schema::Primitive; + + static error_or decode(data& to, int from, int nc_varid){ + int rc{}; + + nc_type nc_vartype{}; + int nc_dimnum{}; + std::array nc_dimids; + + rc = nc_inq_var(from, nc_varid, nullptr, &nc_vartype, &nc_dimnum, &nc_dimids[0], nullptr); + if(rc != NC_NOERR){ + return make_error(); + } + if(nc_vartype != netcdf_primitive_id>::value){ + return make_error(); + } + if(nc_dimnum != 0){ + return make_error(); + } + + typename native_data_type>::type val{}; + rc = nc_get_var(from, nc_varid, &val); + if(rc != NC_NOERR){ + return make_error(); + } + + to.set(val); + + return void_t {}; + } +}; + +template +struct netcdf_decode, ToDecode> { + using Schema = schema::Array; + + template + static error_or decode_level(data& to, int from, int nc_varid, std::array& index, const std::array& count){ + if constexpr ( Level == Dim ){ + int rc{}; + typename native_data_type::type val; + rc = nc_get_vara(from, nc_varid, index.data(), count.data(), &val); + if(rc != NC_NOERR){ + return make_error(); + } + to.at(index).set(val); + }else{ + const std::size_t dim_size = to.get_dim_size(Level); + for(index[Level] = 0; index[Level] < dim_size; ++index[Level]){ + auto eov = decode_level(to, from, nc_varid, index, count); + if(eov.is_error()){ + return eov; + } + } + } + + return void_t{}; + } + + static error_or decode(data& to, int from, int nc_varid){ + int rc{}; + + nc_type nc_vartype{}; + + int nc_dimnum{}; + std::array nc_dimids; + + rc = nc_inq_var(from, nc_varid, nullptr, &nc_vartype, &nc_dimnum, &nc_dimids[0], nullptr); + if(rc != NC_NOERR){ + return make_error(); + } + if(nc_vartype != netcdf_primitive_id::value){ + return make_error(); + } + if(nc_dimnum != Dim){ + return make_error(); + } + + std::array dims; + std::array count; + for(std::size_t i = 0; i < Dim; ++i){ + rc = nc_inq_dim(from, nc_dimids[i], nullptr, &dims[i]); + if(rc != NC_NOERR){ + return make_error(); + } + count[i] = 1; + } + + to = {dims}; + std::array index; + + return decode_level<0>(to, from, nc_varid, index, count); + } +}; + +template +struct netcdf_decode...>, ToDecode> { + using Schema = schema::Struct...>; + + template + static error_or decode_member(data& to, int from){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + /** + * We have to ask for the internal id of the netcdf structure + */ + if constexpr (netcdf_is_group::value){ + int nc_group_id{}; + int rc {}; + rc = nc_inq_ncid(from, Literal.data.data(), &nc_group_id); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_decode::decode(to.template get(), nc_group_id); + if(eov.is_error()){ + return eov; + } + }else{ + int nc_varid{}; + int rc {}; + rc = nc_inq_varid(from, Literal.data.data(), &nc_varid); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_decode::decode(to.template get(), from, nc_varid); + if(eov.is_error()){ + return eov; + } + } + + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = decode_member(to, from); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } + + + static error_or decode(data& to, int from){ + if constexpr (sizeof...(T) > 0){ + auto eov = decode_member<0>(to, from); + return eov; + } + + return void_t{}; + } +}; +} +} diff --git a/modules/codec/.nix/derivation.nix b/modules/codec/.nix/derivation.nix new file mode 100644 index 0000000..bcf2be7 --- /dev/null +++ b/modules/codec/.nix/derivation.nix @@ -0,0 +1,28 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-codec"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + buildInputs = [ + forstio.core + ]; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/codec/SConscript b/modules/codec/SConscript new file mode 100644 index 0000000..c038d42 --- /dev/null +++ b/modules/codec/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +codec_env = env.Clone(); + +codec_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +codec_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += codec_env.sources; +env.headers += codec_env.headers; + +## Shared lib +objects_shared = [] +codec_env.add_source_files(objects_shared, codec_env.sources, shared=True); +codec_env.library_shared = codec_env.SharedLibrary('#build/forstio-codec', [objects_shared]); + +## Static lib +objects_static = [] +codec_env.add_source_files(objects_static, codec_env.sources, shared=False); +codec_env.library_static = codec_env.StaticLibrary('#build/forstio-codec', [objects_static]); + +# Set Alias +env.Alias('library_codec', [codec_env.library_shared, codec_env.library_static]); + +env.targets += ['library_codec']; + +# Install +env.Install('$prefix/lib/', [codec_env.library_shared, codec_env.library_static]); +env.Install('$prefix/include/forstio/codec/', [codec_env.headers]); diff --git a/modules/codec/SConstruct b/modules/codec/SConstruct new file mode 100644 index 0000000..0d7b7c6 --- /dev/null +++ b/modules/codec/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-core']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/codec/args.h b/modules/codec/args.h new file mode 100644 index 0000000..6bb75a2 --- /dev/null +++ b/modules/codec/args.h @@ -0,0 +1,123 @@ +#pragma once + +#include "schema.h" + +namespace saw { +namespace encode { +struct Args {}; +} + +namespace schema { +template +using Args = Struct< + Member, + Member, + Member +>; +} + +template +class data { + static_assert(always_false,"Not supported. Use the schema::Args alias or check how it's defined."); +}; + +template +class data...>, schema::Tuple>, encode::Args> { +private: + char** argv_; + int argc_; +public: + data(char** argv, int argc): + argv_{argv}, + argc_{argc} + {} + + size_t size() const { + if(argc_ < 0){ + return 0; + } + + static_assert(sizeof(int) <= sizeof(size_t), "size_t is smaller than int"); + + return static_cast(argc_); + } + + std::string_view arg_view(size_t i){ + if(i < size()){ + return std::string_view{argv_[i]}; + } + return ""; + } +}; + +namespace impl { +template +struct args_decode { + static_assert(always_false, "Not supported"); +}; + +template +struct args_decode...>,schema::Tuple>,ToDec> { + using Schema = schema::Args< + schema::Struct...>, + schema::Tuple + >; + + static error_or decode(data& from, data& to){ + if(from.size() == 0){ + return make_error(); + } + + to.get<"program">().set(std::string{from.arg_view(0)}); + std::size_t tuple_pos = 0; + for(size_t i = 1; i < from.size(); ++i){ + auto view = from.arg_view(i); + if(view.starts_with("--")){ + view.remove_prefix(std::min(2u, view.size())); + ++i; + + if( i >= from.size() ){ + return make_error(); + } + + auto value_view = from.arg_view(i); + + auto eov = decode_struct_member<0>(to.get<"args">(), view, value_view); + if(eov.is_error()){ + return eov; + } + } else { + auto eov = decode_tuple_member<0>(to.get<"positionals">(), view); + if(eov.is_error()){ + return eov; + } + ++tuple_pos; + } + } + + if(tuple_pos != sizeof...(Tup)){ + return make_error(); + } + + return void_t{}; + } +}; +} + +template +class codec { +public: + template + error_or decode(data& from, data& to){ + struct name_and_value { + std::string name; + std::string value; + }; + std::string program; + std::vector navs; + std::vector positionals; + + return void_t{}; + } +}; +} diff --git a/modules/codec/csv.h b/modules/codec/csv.h new file mode 100644 index 0000000..67c2c1d --- /dev/null +++ b/modules/codec/csv.h @@ -0,0 +1,25 @@ +#pragma once + +#include "data.h" + +namespace saw { +namespace encode { +struct Csv {}; +} + +template +class codec { +public: + template + static error_or encode(const data& from, data& to){ + + return make_error(); + } + + template + static error_or decode(data& from, data& to){ + + return make_error(); + } +}; +} diff --git a/modules/codec/data.h b/modules/codec/data.h new file mode 100644 index 0000000..237ef5a --- /dev/null +++ b/modules/codec/data.h @@ -0,0 +1,370 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include + +#include "schema.h" + +namespace saw { +namespace encode { +struct Native {}; +} +template +class codec; +/* + * Helper for the basic message container, so the class doesn't have to be + * specialized 10 times. + */ +template struct native_data_type; + +template <> +struct native_data_type> { + using type = int8_t; +}; + +template <> +struct native_data_type> { + using type = int16_t; +}; + +template <> +struct native_data_type> { + using type = int32_t; +}; + +template <> +struct native_data_type> { + using type = int64_t; +}; + +template <> +struct native_data_type> { + using type = uint8_t; +}; + +template <> +struct native_data_type> { + using type = uint16_t; +}; + +template <> +struct native_data_type> { + using type = uint32_t; +}; + +template <> +struct native_data_type> { + using type = uint64_t; +}; + +template <> +struct native_data_type> { + using type = float; +}; + +template <> +struct native_data_type> { + using type = double; +}; + +template +class data { +private: + static_assert(always_false, "Type not supported"); +}; + +template +class data...>, encode::Native> { +private: + std::variant...> value_; +public: + data() = default; + + SAW_DEFAULT_COPY(data); + SAW_DEFAULT_MOVE(data); + + template + void set(data::value, T...>::type, encode::Native> val){ + value_ = std::move(val); + } + + template + data::value, T...>::type, encode::Native>& init(){ + value_ = data::value, T...>::type, encode::Native>{}; + return get(); + } + + template + bool holds_alternative() const { + return (parameter_key_pack_index::value == value_.index()); + } + + template + data::value, T...>::type, encode::Native>& get(){ + return std::get::value>(value_); + } + + template + const data::value, T...>::type, encode::Native>& get() const{ + return std::get::value>(value_); + } +}; + +template +class data...>, encode::Native> { +private: + std::tuple...> value_; +public: + data() = default; + SAW_DEFAULT_COPY(data); + SAW_DEFAULT_MOVE(data); + + template + data< + typename parameter_pack_type< + parameter_key_pack_index< + literal, literals... + >::value + , T...>::type + , encode::Native>& get(){ + return std::get::value>(value_); + } + + template + const data< + typename parameter_pack_type< + parameter_key_pack_index< + literal, literals... + >::value + , T...>::type + , encode::Native>& get() const { + return std::get::value>(value_); + } + + constexpr size_t size() const { + return sizeof...(T); + } +}; + +template +class data, encode::Native> { +private: + std::tuple...> value_; +public: + data() = default; + SAW_DEFAULT_COPY(data); + SAW_DEFAULT_MOVE(data); + + template + data::type, encode::Native>& get(){ + return std::get(value_); + } + + template + const data::type, encode::Native>& get() const{ + return std::get(value_); + } + + constexpr size_t size() const { + return sizeof...(T); + } +}; + +template +class data, encode::Native> { + private: + std::array dims_; + std::vector> value_; + + std::size_t get_full_size() const { + std::size_t s = 1; + + for(std::size_t iter = 0; iter < Dim; ++iter){ + assert(dims_.at(iter) > 0); + s *= dims_.at(iter); + } + + return s; + } + public: + data() = default; + SAW_DEFAULT_COPY(data); + SAW_DEFAULT_MOVE(data); + + data(const std::array& i): + dims_{i}, + value_{} + { + value_.resize(get_full_size()); + } + + template + data(Dims... size_): + data{{size_...}} + { + static_assert(sizeof...(Dims)==Dim, "Argument size must be equal to the Dimension"); + } + + data& at(const std::array& ind){ + return value_.at(this->get_flat_index(ind)); + } + + const data& at(const std::array& ind) const { + return value_.at(this->get_flat_index(ind)); + } + + template + data& at(Dims... i){ + return value_.at(this->get_flat_index({i...})); + } + + template + const data& at(Dims... i) const { + return value_.at(this->get_flat_index({i...})); + } + + std::size_t get_dim_size(std::size_t i) const { + return dims_.at(i); + } + + size_t size() const { return value_.size();} + +private: + std::size_t get_flat_index(const std::array& i) const { + std::size_t s = 0; + + std::size_t stride = 1; + + for(std::size_t iter = 0; iter < Dim; ++iter){ + assert(i.at(iter) < dims_.at(iter)); + s += i.at(iter) * stride; + stride *= dims_.at(iter); + } + + return s; + } +}; + +template +class data, encode::Native> { +private: + //using inner_type = std::array, multiply_helper::value>; + //std::unique_ptr value_; + std::vector> value_; + +public: + data(){ + value_.resize(ct_multiply::value); + } + + data& at(const std::array& ind){ + return value_.at(this->get_flat_index(ind)); + } + + const data& at(const std::array& ind) const { + return value_.at(this->get_flat_index(ind)); + } + + template + data& at(Dims... i) { + return value_.at(this->get_flat_index({i...})); + } + + template + const data& at(Dims... i) const { + return value_.at(this->get_flat_index({i...})); + } + + template + std::size_t get_dim_size() const { + return parameter_pack_value::value; + } +private: + std::size_t get_flat_index(const std::array& i) const { + std::size_t s = 0; + + std::size_t stride = 1; + + constexpr static std::array dims_{D...}; + + for(std::size_t iter = 0; iter < sizeof...(D); ++iter){ + assert(i.at(iter) < dims_.at(iter)); + s += i.at(iter) * stride; + stride *= dims_.at(iter); + } + + return s; + } +}; + +template<> +class data { +private: + std::string value_; +public: + data() = default; + SAW_DEFAULT_COPY(data); + SAW_DEFAULT_MOVE(data); + + data(std::string value__):value_{std::move(value__)}{} + data(std::size_t size_){ + value_.resize(size_); + } + + std::size_t size() const { + return value_.size(); + } + + void set(std::string str){ + value_ = std::move(str); + } + + char& at(size_t i) { + return value_.at(i); + } + + const char& at(size_t i) const { + return value_.at(i); + } + + char get_at(size_t i) const{ + return value_.at(i); + } + + void set_at(size_t i, char val){ + value_.at(i) = val; + } + + bool operator==(const std::string_view& val)const{ + return value_ == val; + } +}; + +template +class data, encode::Native> { +private: + typename native_data_type>::type value_; +public: + data():value_{{}}{}; + SAW_DEFAULT_COPY(data); + SAW_DEFAULT_MOVE(data); + + data(typename native_data_type>::type value__): + value_{std::move(value__)}{} + + void set(typename native_data_type>::type val){ + value_ = val; + } + + typename native_data_type>::type get() const {return value_;} +}; + + +} diff --git a/modules/codec/forst.h b/modules/codec/forst.h new file mode 100644 index 0000000..7e8fbf0 --- /dev/null +++ b/modules/codec/forst.h @@ -0,0 +1,32 @@ +#pragma once + +#include "data.h" + +namespace saw { +namespace encode { +struct KelForst {}; +} +} + +#include "forst.tmpl.hpp" + +namespace saw { +class data { +private: + own buff_; +public: + data(own buff): + buff_{std::move(buff)} + {} +}; + +template +class data...>, encode::KelForst> { +private: + own buff_; +public: + data(own buff): + buff_{std::move(buff)} + {} +}; +} diff --git a/modules/codec/forst.tmpl.h b/modules/codec/forst.tmpl.h new file mode 100644 index 0000000..30d18ef --- /dev/null +++ b/modules/codec/forst.tmpl.h @@ -0,0 +1,7 @@ +namespace saw { +namespace impl { +struct forst_decode { + +}; +} +} diff --git a/modules/codec/interface.h b/modules/codec/interface.h new file mode 100644 index 0000000..b1422a9 --- /dev/null +++ b/modules/codec/interface.h @@ -0,0 +1,103 @@ +#pragma once + +#include +#include "schema.h" +#include "data.h" + +namespace saw { +template +class function; + +template +class function, Encode, Func> { +private: + Func func_; +public: + function(Func func): + func_{std::move(func)} + {} + + error_or> call(data req){ + return func_(std::move(req)); + } +}; + +template +class interface; + +template +class interface, Names>...>, Encode, Funcs...> { +private: + std::tuple, Encode, Funcs>...> funcs_; +public: + interface(function, Encode, Funcs>... funcs): + funcs_{std::move(funcs)...} + {} + + /** + * Get the function based on the string literal matching the position in the tuple + */ + template + function< + schema::Function< + typename parameter_pack_type< + parameter_key_pack_index< + Lit, Names... + >::value + , Requests...>::type + , + typename parameter_pack_type< + parameter_key_pack_index< + Lit, Names... + >::value + , Responses...>::type + > + , + Encode + , + typename parameter_pack_type< + parameter_key_pack_index< + Lit, Names... + >::value + , Funcs...>::type + >& get(){ + return std::get::value>(funcs_); + } + + template + error_or::value + , Responses...>::type + , Encode>> call( + data< + typename parameter_pack_type< + parameter_key_pack_index< + Lit, Names... + >::value + , Requests...>::type + , Encode> req + ){ + return get().call(std::move(req)); + } + +}; + +template +struct function_factory { + template + static function create(Func func){ + return function{std::move(func)}; + } +}; + +template +struct interface_factory { + template + static interface create(Func... func){ + return interface{std::move(func)...}; + } +}; +} diff --git a/modules/codec/rpc.h b/modules/codec/rpc.h new file mode 100644 index 0000000..b21ecf8 --- /dev/null +++ b/modules/codec/rpc.h @@ -0,0 +1,26 @@ +#pragma once + +namespace saw { +template +class remote { + static_assert(always_false, "Type of remote not supported"); +}; + +template +class rpc_client { + template + struct request { + private: + std::tuple> ids_; + public: + error_or> wait(); + }; + + template + request request_data(id... data); +}; + +template +class rpc_server { +}; +} diff --git a/modules/codec/schema.h b/modules/codec/schema.h new file mode 100644 index 0000000..a8494fe --- /dev/null +++ b/modules/codec/schema.h @@ -0,0 +1,109 @@ +#pragma once + +#include +#include + +namespace saw { +namespace schema { +// NOLINTBEGIN +template struct Member {}; + +template struct Struct { + static_assert( + always_false, + "This schema template doesn't support this type of template argument"); +}; + +template +struct Struct...> {}; + +template struct Union { + static_assert( + always_false, + "This schema template doesn't support this type of template argument"); +}; + +template +struct Union...> {}; + +template struct Array {}; + +template struct is_array { + constexpr static bool value = false; +}; + +template struct is_array> { + constexpr static bool value = true; +}; + +template struct FixedArray {}; + +template struct Tuple {}; + +/** + * This acts as a separator of different encodings being mashed together + * For example we can transport any base64 encodings in JSON + * + * using WrappedExample = schema::Tuple< + * schema::Wrapper + * >; + * + * data ex_data; + */ +template +class Wrapper {}; + +struct String {}; + +struct SignedInteger {}; +struct UnsignedInteger {}; +struct FloatingPoint {}; + +template struct Primitive { + static_assert(((std::is_same_v || + std::is_same_v)&&(N == 1 || N == 2 || + N == 4 || N == 8)) || + (std::is_same_v && (N == 4 || N == 8)), + "Primitive Type is not supported"); +}; + +using Int8 = Primitive; +using Int16 = Primitive; +using Int32 = Primitive; +using Int64 = Primitive; + +using UInt8 = Primitive; +using UInt16 = Primitive; +using UInt32 = Primitive; +using UInt64 = Primitive; + +using Float32 = Primitive; +using Float64 = Primitive; + +/** + * Classes allowing to distinguish Ints from VarInts + */ +template +struct VariableLengthPrimitive {}; + +using VarInt = VariableLengthPrimitive; +using VarLong = VariableLengthPrimitive; + +/** + * Classes enabling Rpc calls + */ +template +struct Function {}; + +template struct Interface { + static_assert( + always_false, + "This schema template doesn't support this type of template argument"); +}; + +template +struct Interface,Names>...> {}; + +// NOLINTEND +} // namespace schema +} // namespace saw diff --git a/modules/codec/schema_hash.h b/modules/codec/schema_hash.h new file mode 100644 index 0000000..5690166 --- /dev/null +++ b/modules/codec/schema_hash.h @@ -0,0 +1,105 @@ +#pragma once + +#include "schema.h" + +namespace saw { +struct schema_hash_combine { + static constexpr uint64_t apply(uint64_t seed, uint64_t v){ + return (seed ^ v) * 1099511628211u; + + + return seed ^( std::hash{}(v) + 0x9e3779b9 + (seed<<6) + (seed >> 2)); + } +}; + +template +struct schema_hash { + static_assert(always_false, "Not schema_hashable"); +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 0u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 1u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 2u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 3u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 4u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 5u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 6u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 7u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 8u; +}; + +template<> +struct schema_hash> { + static constexpr uint64_t base_value = 9u; +}; + +template +struct schema_hash> { + static constexpr uint64_t base_value = 10u; +}; + +template +struct schema_hash...>> { + static constexpr uint64_t base_value = 11u; +}; + +template +struct schema_hash...>> { + static constexpr uint64_t base_value = 12u; +}; + +template +struct schema_hash> { + static constexpr uint64_t base_value = 13u; +}; + +template<> +struct schema_hash { + static constexpr uint64_t base_value = 14u; +}; + +template +struct schema_hash> { + static constexpr uint64_t base_value = 15u; +}; + +template +struct schema_hash> { + static constexpr uint64_t base_value = 16u; +}; + +} diff --git a/modules/codec/schema_stringify.h b/modules/codec/schema_stringify.h new file mode 100644 index 0000000..a82081a --- /dev/null +++ b/modules/codec/schema_stringify.h @@ -0,0 +1,118 @@ +#pragma once + +#include "schema.h" + +#include +#include + +namespace saw { +template +struct schema_stringify { + static_assert(always_false, "Not supported"); +}; + +template +struct schema_stringify> { + static void apply(std::stringstream& iss) { + iss << "saw::schema::Array<"; + schema_stringify::apply(iss); + iss << ","; + iss << ct_convert_to_digits::literal.view(); + iss << ">"; + } +}; + +template +struct schema_stringify> { + static void apply(std::stringstream& iss) { + iss << "saw::schema::Primitive<"; + schema_stringify::apply(iss); + iss << ","; + iss << ct_convert_to_digits::literal.view(); + iss << ">"; + } +}; + +template<> +struct schema_stringify { + static void apply(std::stringstream& iss) { + iss << "saw:schema::SignedInteger"; + } +}; + +template<> +struct schema_stringify { + static void apply(std::stringstream& iss) { + iss << "saw:schema::UnsignedInteger"; + } +}; + +template<> +struct schema_stringify { + static void apply(std::stringstream& iss) { + iss << "saw:schema::FloatingPoint"; + } +}; + +template +struct schema_stringify_member { + static void apply(std::stringstream& iss) { + (void)iss; + } +}; + +template +struct schema_stringify_member, TL...> { + + static void apply(std::stringstream& iss) { + iss << "saw::schema::Member<"; + schema_stringify::apply(iss); + iss << ",\""; + iss << Name.view(); + iss << "\">"; + if constexpr ( sizeof...(TL) > 0){ + iss << ","; + schema_stringify_member::apply(iss); + } + } +}; + +template +struct schema_stringify...>> { + static void apply(std::stringstream& iss) { + iss << "saw::schema::Struct<"; + schema_stringify_member...>::apply(iss); + iss << ">"; + } +}; + +template +struct schema_stringify...>> { + static void apply(std::stringstream& iss) { + iss << "saw::schema::Union<"; + schema_stringify_member...>::apply(iss); + iss << ">"; + } +}; + +template +struct schema_stringify> { + static void apply(std::stringstream& iss){ + iss << "saw::schema::Function<"; + schema_stringify::apply(iss); + iss << ","; + schema_stringify::apply(iss); + iss << ">"; + } +}; + +template +struct schema_stringify...>>{ + static void apply(std::stringstream& iss){ + iss << "saw::schema::Interface<"; + schema_stringify_member...>::apply(iss); + iss << ">"; + } +}; + +} diff --git a/modules/codec/simple.h b/modules/codec/simple.h new file mode 100644 index 0000000..8760754 --- /dev/null +++ b/modules/codec/simple.h @@ -0,0 +1,389 @@ +#pragma once + +#include "data.h" +#include "stream_value.h" + +#include +#include + +namespace saw { +namespace encode { +struct KelSimple {}; +} + +template +class data { +private: + ring_buffer buffer_; +public: + data() = default; + + buffer& get_buffer(){ + return buffer_; + } +}; + +namespace impl { +template +class kelsimple_encode { + static_assert(always_false, "This schema type is not being handled by the kelsimple encoding."); +}; + +template +struct kelsimple_encode, FromEnc> { + static error_or encode(const data, FromEnc>& from, buffer& to){ + auto eov = stream_value>::encode(from.get(), to); + return eov; + } +}; + +template +struct kelsimple_encode, FromEnc> { + template + static error_or encode_level(const data, FromEnc>& from, buffer& to, std::array& index){ + if constexpr (Dim == Level){ + return kelsimple_encode::encode(from.at(index), to); + } else { + const std::size_t dim_size = from.get_dim_size(Level); + for(index[Level] = 0; (index.at(Level) < dim_size); ++index[Level]){ + auto eov = encode_level(from, to, index); + if(eov.is_error()){ + return eov; + } + } + } + return void_t{}; + } + + static error_or encode(const data, FromEnc>& from, buffer& to){ + { + for(uint64_t i = 0; i < Dim; ++i){ + auto eov = stream_value::encode(from.get_dim_size(i), to); + if(eov.is_error()){ + return eov; + } + } + } + { + std::array index; + std::fill(index.begin(), index.end(), 0); + + return encode_level<0>(from, to, index); + } + return void_t{}; + } +}; + +template +struct kelsimple_encode...>,FromEnc> { + template + static error_or encode_member(const data...>, FromEnc>& from, buffer& to){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + auto eov = kelsimple_encode::encode(from.template get(), to); + if(eov.is_error()){ + return eov; + } + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = encode_member(from, to); + if(eov.is_error()){ + return eov; + } + } + return void_t{}; + } + + static error_or encode(const data...>, FromEnc>& from, buffer& to){ + return encode_member<0>(from, to); + } +}; + +template +struct kelsimple_encode...>,FromEnc> { + template + static error_or encode_member(const data...>, FromEnc>& from, buffer& to){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + if (from.template holds_alternative()) { + { + auto eov = stream_value::encode(static_cast(i), to); + if(eov.is_error()){ + return eov; + } + } + { + auto eov = kelsimple_encode::encode(from.template get(), to); + if(eov.is_error()){ + return eov; + } + } + } + + if constexpr ( (i+1) < sizeof...(T) ){ + auto eov = encode_member(from, to); + if(eov.is_error()){ + return eov; + } + } + return void_t{}; + } + + static error_or encode(const data...>, FromEnc>& from, buffer& to){ + return encode_member<0>(from, to); + } +}; + +template +struct kelsimple_encode, FromEnc> { + template + static error_or encode_member(const data, FromEnc>& from, buffer& to){ + using Type = typename parameter_pack_type::type; + { + auto eov = kelsimple_encode::encode(from.template get(), to); + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = encode_member(from, to); + if(eov.is_error()){ + return eov; + } + } + return void_t{}; + } + + static error_or encode(const data, FromEnc>& from, buffer& to){ + return encode_member<0>(from, to); + } +}; + +template +struct kelsimple_encode { + static error_or encode(const data& from, buffer& to){ + const auto str_size = from.size(); + typename native_data_type::type str_len = static_cast(str_size); + { + auto eov = stream_value::encode(str_len, to); + if(eov.is_error()){ + return eov; + } + } + + for(std::size_t i = 0; i < str_size; ++i){ + auto eov = stream_value::encode(from.at(i), to); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +template +class kelsimple_decode { + static_assert(always_false, "This schema type is not being handled by the kelsimple encoding."); +}; + +template +struct kelsimple_decode, FromEnc> { + static error_or decode(buffer& from, data, FromEnc>& to){ + typename native_data_type>::type val{}; + auto eov = stream_value>::decode(val, from); + if (eov.is_value()) { + to.set(val); + } + return eov; + } + +}; + +template +struct kelsimple_decode, FromEnc> { + template + static error_or decode_level(buffer& from, data, FromEnc>& to, std::array& index){ + if constexpr (Level == Dim){ + return kelsimple_decode::decode(from, to.at(index)); + }else{ + const std::size_t dim_size = to.get_dim_size(Level); + for(index[Level] = 0; index[Level] < dim_size; ++index[Level]){ + auto eov = decode_level(from, to, index); + if(eov.is_error()){ + return eov; + } + } + } + return void_t{}; + } + + static error_or decode(buffer& from, data, FromEnc>& to){ + { + std::array dims{}; + for(std::size_t i = 0; i < Dim; ++i){ + uint64_t val{}; + auto eov = stream_value::decode(val, from); + if(eov.is_error()){ + return eov; + } + dims.at(i) = static_cast(val); + } + to = data,FromEnc>{dims}; + } + { + std::array index{}; + return decode_level<0>(from, to, index); + } + return void_t{}; + } +}; +template +struct kelsimple_decode...>,FromEnc> { + template + static error_or decode_member(buffer& from, data...>, FromEnc>& to){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + auto eov = kelsimple_decode::decode(from, to.template get()); + if(eov.is_error()){ + return eov; + } + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = decode_member(from, to); + if(eov.is_error()){ + return eov; + } + } + return void_t{}; + + } + static error_or decode(buffer& from, data...>, FromEnc>& to){ + return decode_member<0>(from, to); + } + +}; + +template +struct kelsimple_decode...>,FromEnc> { + template + static error_or decode_member(buffer& from, data...>, FromEnc>& to, uint64_t val){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + + if( i == val ){ + to.template set(data{}); + auto eov = kelsimple_decode::decode(from, to.template get()); + if(eov.is_error()){ + return eov; + } + return void_t{}; + } + + if constexpr ((i+1) < sizeof...(T)){ + auto eov = decode_member(from, to, val); + if(eov.is_error()){ + return eov; + } + } + return void_t{}; + + } + static error_or decode(buffer& from, data...>, FromEnc>& to){ + uint64_t val{}; + auto eov = stream_value::decode(val, from); + if(eov.is_error()){ + return eov; + } + if ( val >= sizeof...(T) ){ + return make_error(); + } + return decode_member<0>(from, to, val); + } + +}; + +template +struct kelsimple_decode,FromEnc> { + template + static error_or decode_member(buffer& from, data, FromEnc>& to){ + using Type = typename parameter_pack_type::type; + { + auto eov = kelsimple_decode::decode(from, to.template get()); + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = decode_member(from, to); + if(eov.is_error()){ + return eov; + } + } + return void_t{}; + + } + static error_or decode(buffer& from, data, FromEnc>& to){ + return decode_member<0>(from, to); + } + +}; +template +struct kelsimple_decode { + static error_or decode(buffer& from, data& to){ + { + uint64_t val{}; + auto eov = stream_value::decode(val, from); + if(eov.is_error()){ + return eov; + } + to = data{val}; + } + const std::size_t str_size = to.size(); + for(std::size_t i = 0; i < str_size; ++i){ + int8_t val{}; + auto eov = stream_value::decode(val, from); + if(eov.is_error()){ + return eov; + } + to.set_at(i, val); + } + return void_t{}; + } +}; + +} + +template +class codec { +public: + struct config { + size_t depth = 16; + size_t length = 1024; + }; +private: + config cfg_; +public: + codec() = default; + + SAW_FORBID_COPY(codec); + SAW_DEFAULT_MOVE(codec); + + template + error_or encode(const data& from_enc, data& to_enc){ + buffer_view buff_v{to_enc.get_buffer()}; + + auto eov = impl::kelsimple_encode::encode(from_enc, buff_v); + + to_enc.get_buffer().write_advance(buff_v.write_offset()); + + return eov; + } + + template + error_or decode(data& from_dec, data& to){ + buffer_view buff_v{from_dec.get_buffer()}; + + auto eov = impl::kelsimple_decode::decode(buff_v, to); + + return eov; + } +}; +} diff --git a/modules/codec/stream_value.h b/modules/codec/stream_value.h new file mode 100644 index 0000000..09203cb --- /dev/null +++ b/modules/codec/stream_value.h @@ -0,0 +1,64 @@ +#pragma once + +#include "schema.h" + +#include +#include + +#include +#include + +namespace saw { +/** + * Helper class to encode/decode any primtive type into/from litte endian. + * The shift class does this by shifting bytes. This type of procedure is + * platform independent. So it does not matter if the memory layout is + * little endian or big endian + */ +template class shift_stream_value { + static_assert(always_false, "Shift Stream Value only supports Primitives"); +}; + +template class shift_stream_value> { +public: + inline static error_or decode(typename native_data_type>::type &val, buffer &buff) { + if (buff.read_composite_length() < N) { + return make_error(); + } + + typename native_data_type>::type raw = 0; + + for (size_t i = 0; i < N; ++i) { + raw |= (static_cast>::type>(buff.read(i)) << (i * 8)); + } + + memcpy(&val, &raw, N); + buff.read_advance(N); + + return void_t{}; + } + + inline static error_or encode(const typename native_data_type>::type &val, buffer &buff) { + error err = buff.write_require_length(N); + if (err.failed()) { + return err; + } + + typename native_data_type>::type raw{}; + memcpy(&raw, &val, N); + + for (size_t i = 0; i < N; ++i) { + buff.write(i) = raw >> (i * 8); + } + + buff.write_advance(N); + + return void_t{}; + } + + inline static size_t size() { return N; } +}; + +template using stream_value = shift_stream_value; + +} // namespace saw diff --git a/modules/core/.nix/derivation.nix b/modules/core/.nix/derivation.nix new file mode 100644 index 0000000..1618651 --- /dev/null +++ b/modules/core/.nix/derivation.nix @@ -0,0 +1,21 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +}: + +stdenv.mkDerivation { + pname = "forstio-core"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/core/SConscript b/modules/core/SConscript new file mode 100644 index 0000000..02bd050 --- /dev/null +++ b/modules/core/SConscript @@ -0,0 +1,42 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +core_env = env.Clone(); + +core_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +core_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += core_env.sources; +env.headers += core_env.headers; + +## Shared lib +objects_shared = [] +core_env.add_source_files(objects_shared, core_env.sources, shared=True); +core_env.library_shared = core_env.SharedLibrary('#build/forstio-core', [objects_shared]); + +## Static lib +objects_static = [] +core_env.add_source_files(objects_static, core_env.sources, shared=False); +core_env.library_static = core_env.StaticLibrary('#build/forstio-core', [objects_static]); + +# Set Alias +env.Alias('library_core', [core_env.library_shared, core_env.library_static]); + +env.targets += ['library_core']; + +# Install +env.Install('$prefix/lib/', [core_env.library_shared, core_env.library_static]); +env.Install('$prefix/include/forstio/core/', [core_env.headers]); + +# Test +# Export('core_env'); +# SConscript('tests/SConscript'); diff --git a/modules/core/SConstruct b/modules/core/SConstruct new file mode 100644 index 0000000..865d131 --- /dev/null +++ b/modules/core/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=[]) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/core/array.h b/modules/core/array.h new file mode 100644 index 0000000..f82b8d6 --- /dev/null +++ b/modules/core/array.h @@ -0,0 +1,96 @@ +#pragma once + +#include + +namespace saw { +/** + * Array container avoiding exceptions + */ +template +class array { +private: + std::vector data_; +public: + array() = default; + + SAW_FORBID_COPY(array); + + T& at(size_t i) noexcept { + return data_.at(i); + } + + const T& at(size_t i) noexcept const { + return data_.at(i); + } + + size_t size() noexcept const { + return data_.size(); + } + + T& front() noexcept { + return data_.front(); + } + + const T& front() noexcept const { + return data_.front(); + } + + T& back() noexcept { + return data_.back(); + } + + const T& back() noexcept const { + return data_.back(); + } + + error_or push(T val) noexcept { + try{ + data_.push_back(std::move(val)); + }catch(std::exception& e){ + return make_error(); + } + + return void_t{}; + } + + error_or pop() noexcept { + try{ + data_.pop_back(); + }catch(std::exception& e){ + return make_error(); + } + + return void_t{}; + } + + error_or resize(size_t i) noexcept { + try{ + data_.resize(i); + }catch(std::exception& e){ + return make_error(); + } + + return void_t{}; + } + + error_or reserve(size_t i) noexcept { + try{ + data_.reserve(i); + }catch(std::exception& e){ + return make_error(); + } + + return void_t{}; + } + + error_or shrink_to_fit() noexcept { + try{ + data_.shrink_to_fit(); + }catch(std::exception& e){ + return make_error(); + } + + return void_t{}; + } +}; +} diff --git a/modules/core/buffer.cpp b/modules/core/buffer.cpp new file mode 100644 index 0000000..15f4cae --- /dev/null +++ b/modules/core/buffer.cpp @@ -0,0 +1,436 @@ +#include "buffer.h" + +#include +#include +#include +#include +#include + +namespace saw { +error buffer::push(const uint8_t &value) { + size_t write_remain = write_composite_length(); + if (write_remain > 0) { + write() = value; + write_advance(1); + } else { + return make_error(); + } + return no_error(); +} + +error buffer::push(const uint8_t &buffer, size_t size) { + error error = write_require_length(size); + if (error.failed()) { + return error; + } + const uint8_t *buffer_ptr = &buffer; + while (size > 0) { + size_t segment = std::min(write_segment_length(), size); + memcpy(&write(), buffer_ptr, segment); + write_advance(segment); + size -= segment; + buffer_ptr += segment; + } + return no_error(); +} + +error buffer::pop(uint8_t &value) { + if (read_composite_length() > 0) { + value = read(); + read_advance(1); + } else { + return make_error(); + } + return no_error(); +} + +error buffer::pop(uint8_t &buffer, size_t size) { + if (read_composite_length() >= size) { + uint8_t *buffer_ptr = &buffer; + while (size > 0) { + size_t segment = std::min(read_segment_length(), size); + memcpy(buffer_ptr, &read(), segment); + read_advance(segment); + size -= segment; + buffer_ptr += segment; + } + } else { + return make_error(); + } + return no_error(); +} + +/* +std::string buffer::to_hex() const { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + for (size_t i = 0; i < read_composite_length(); ++i) { + oss << std::setw(2) << (uint16_t)read(i); + if ((i + 1) < read_composite_length()) { + oss << ((i % 4 == 3) ? '\n' : ' '); + } + } + return oss.str(); +} +*/ + +std::string convert_to_string(const buffer& buff){ + std::ostringstream oss; + for (size_t i = 0; i < buff.read_composite_length(); ++i) { + oss << buff.read(i); + } + return oss.str(); +} + +buffer_view::buffer_view(buffer &buffer) + : buffer_{&buffer}, read_offset_{0}, write_offset_{0} {} + +size_t buffer_view::read_position() const { + return read_offset_ + buffer_->read_position(); +} + +size_t buffer_view::read_composite_length() const { + assert(read_offset_ <= buffer_->read_composite_length()); + if (read_offset_ > buffer_->read_composite_length()) { + return 0; + } + + return buffer_->read_composite_length() - read_offset_; +} + +size_t buffer_view::read_segment_length(size_t offset) const { + size_t off = offset + read_offset_; + assert(off <= buffer_->read_composite_length()); + if (off > buffer_->read_composite_length()) { + return 0; + } + + return buffer_->read_segment_length(off); +} + +void buffer_view::read_advance(size_t bytes) { + size_t offset = bytes + read_offset_; + assert(offset <= buffer_->read_composite_length()); + if (offset > buffer_->read_composite_length()) { + read_offset_ += buffer_->read_composite_length(); + return; + } + + read_offset_ += bytes; +} + +uint8_t &buffer_view::read(size_t i) { + size_t pos = i + read_offset_; + + assert(pos < buffer_->read_composite_length()); + + return buffer_->read(pos); +} + +const uint8_t &buffer_view::read(size_t i) const { + size_t pos = i + read_offset_; + + assert(pos < buffer_->read_composite_length()); + + return buffer_->read(pos); +} + +size_t buffer_view::write_position() const { + return write_offset_ + buffer_->write_position(); +} + +size_t buffer_view::write_composite_length() const { + assert(write_offset_ <= buffer_->write_composite_length()); + if (write_offset_ > buffer_->write_composite_length()) { + return 0; + } + + return buffer_->write_composite_length() - write_offset_; +} + +size_t buffer_view::write_segment_length(size_t offset) const { + size_t off = offset + write_offset_; + assert(off <= buffer_->write_composite_length()); + if (off > buffer_->write_composite_length()) { + return 0; + } + + return buffer_->write_segment_length(off); +} + +void buffer_view::write_advance(size_t bytes) { + size_t offset = bytes + write_offset_; + assert(offset <= buffer_->write_composite_length()); + if (offset > buffer_->write_composite_length()) { + write_offset_ += buffer_->write_composite_length(); + return; + } + + write_offset_ += bytes; +} + +uint8_t &buffer_view::write(size_t i) { + size_t pos = i + write_offset_; + + assert(pos < buffer_->write_composite_length()); + + return buffer_->write(pos); +} + +const uint8_t &buffer_view::write(size_t i) const { + size_t pos = i + write_offset_; + + assert(pos < buffer_->write_composite_length()); + + return buffer_->write(pos); +} + +error buffer_view::write_require_length(size_t bytes) { + return buffer_->write_require_length(bytes + write_offset_); +} + +size_t buffer_view::read_offset() const { return read_offset_; } + +size_t buffer_view::write_offset() const { return write_offset_; } + +ring_buffer::ring_buffer() : read_position_{0}, write_position_{0} { + buffer_.resize(RING_BUFFER_MAX_SIZE); +} + +ring_buffer::ring_buffer(size_t size) : read_position_{0}, write_position_{0} { + buffer_.resize(size); +} + +size_t ring_buffer::read_position() const { return read_position_; } + +/* + * If write is ahead of read it is a simple distance, but if read ist ahead of + * write then there are two segments + * + */ +size_t ring_buffer::read_composite_length() const { + return write_position() < read_position() + ? buffer_.size() - (read_position() - write_position()) + : (write_reached_read_ ? buffer_.size() + : write_position() - read_position()); +} + +/* + * If write is ahead then it's the simple distance again. If read is ahead it's + * until the end of the buffer/segment + */ +size_t ring_buffer::read_segment_length(size_t offset) const { + size_t read_composite = read_composite_length(); + assert(offset <= read_composite); + offset = std::min(offset, read_composite); + size_t remaining = read_composite - offset; + + size_t read_offset = read_position() + offset; + read_offset = read_offset >= buffer_.size() ? read_offset - buffer_.size() + : read_offset; + + // case 1 write is located before read and reached read + // then offset can be used normally + // case 2 write is located at read, but read reached write + // then it is set to zero by readCompositeLength() + // case 3 write is located after read + // since std::min you can use simple subtraction + if (write_position() < read_offset) { + return buffer_.size() - read_offset; + } + + if (write_position() == read_offset) { + if (remaining > 0) { + return buffer_.size() - read_offset; + } else { + return 0; + } + } + + return write_position() - read_offset; +} + +void ring_buffer::read_advance(size_t bytes) { + size_t read_composite = read_composite_length(); + + assert(bytes <= read_composite); + bytes = std::min(bytes, read_composite); + size_t advanced = read_position_ + bytes; + read_position_ = advanced >= buffer_.size() ? advanced - buffer_.size() + : advanced; + write_reached_read_ = bytes > 0 ? false : write_reached_read_; +} + +uint8_t &ring_buffer::read(size_t i) { + assert(i < read_composite_length()); + size_t pos = read_position_ + i; + pos = pos >= buffer_.size() ? pos - buffer_.size() : pos; + return buffer_[pos]; +} + +const uint8_t &ring_buffer::read(size_t i) const { + assert(i < read_composite_length()); + size_t pos = read_position_ + i; + pos = pos >= buffer_.size() ? pos - buffer_.size() : pos; + return buffer_[pos]; +} + +size_t ring_buffer::write_position() const { return write_position_; } + +size_t ring_buffer::write_composite_length() const { + return read_position() > write_position() + ? (read_position() - write_position()) + : (write_reached_read_ + ? 0 + : buffer_.size() - (write_position() - read_position())); +} + +size_t ring_buffer::write_segment_length(size_t offset) const { + size_t write_composite = write_composite_length(); + assert(offset <= write_composite); + offset = std::min(offset, write_composite); + + size_t write_offset = write_position() + offset; + write_offset = write_offset >= buffer_.size() + ? write_offset - buffer_.size() + : write_offset; + + if (read_position_ > write_offset) { + return read_position_ - write_offset; + } + + if (write_reached_read_) { + return 0; + } + + return buffer_.size() - write_offset; +} + +void ring_buffer::write_advance(size_t bytes) { + assert(bytes <= write_composite_length()); + size_t advanced = write_position_ + bytes; + write_position_ = advanced >= buffer_.size() ? advanced - buffer_.size() + : advanced; + + write_reached_read_ = + (write_position_ == read_position_ && bytes > 0 ? true : false); +} + +uint8_t &ring_buffer::write(size_t i) { + assert(i < write_composite_length()); + size_t pos = write_position_ + i; + pos = pos >= buffer_.size() ? pos - buffer_.size() : pos; + return buffer_[pos]; +} + +const uint8_t &ring_buffer::write(size_t i) const { + assert(i < write_composite_length()); + size_t pos = write_position_ + i; + pos = pos >= buffer_.size() ? pos - buffer_.size() : pos; + return buffer_[pos]; +} +/* + Error RingBuffer::increaseSize(size_t size){ + size_t old_size = buffer.size(); + size_t new_size = old_size + size; + buffer.resize(new_size); + if(readPosition() > writePosition() || (readPosition() == + writePosition() && write_reached_read)){ size_t remaining = old_size - + writePosition(); size_t real_remaining = 0; while(remaining > 0){ size_t + segment = std::min(remaining, size); memcpy(&buffer[new_size-segment], + &buffer[old_size-segment], segment); remaining -= segment; size -= segment; + old_size -= segment; + new_size -= segment; + } + } + + return noError(); + } +*/ +error ring_buffer::write_require_length(size_t bytes) { + size_t write_remain = write_composite_length(); + if (bytes > write_remain) { + return make_error(); + } + return no_error(); +} + +array_buffer::array_buffer(size_t size) + : read_position_{0}, write_position_{0} { + buffer_.resize(size); +} + +size_t array_buffer::read_position() const { return read_position_; } + +size_t array_buffer::read_composite_length() const { + return write_position_ - read_position_; +} + +size_t array_buffer::read_segment_length(size_t offset) const { + size_t read_composite = read_composite_length(); + assert(offset <= read_composite); + + offset = std::min(read_composite, offset); + size_t read_offset = read_position_ + offset; + + return write_position_ - read_offset; +} + +void array_buffer::read_advance(size_t bytes) { + assert(bytes <= read_composite_length()); + read_position_ += bytes; +} + +uint8_t &array_buffer::read(size_t i) { + assert(i < read_composite_length()); + + return buffer_[i + read_position_]; +} + +const uint8_t &array_buffer::read(size_t i) const { + assert(i + read_position_ < buffer_.size()); + + return buffer_[i + read_position_]; +} + +size_t array_buffer::write_position() const { return write_position_; } + +size_t array_buffer::write_composite_length() const { + assert(write_position_ <= buffer_.size()); + return buffer_.size() - write_position_; +} + +size_t array_buffer::write_segment_length(size_t offset) const { + assert(write_position_ <= buffer_.size()); + size_t write_composite = write_composite_length(); + + assert(offset <= write_composite); + offset = std::min(write_composite, offset); + size_t write_offset = write_position_ + offset; + + return buffer_.size() - write_offset; +} + +void array_buffer::write_advance(size_t bytes) { + assert(bytes <= write_composite_length()); + write_position_ += bytes; +} + +uint8_t &array_buffer::write(size_t i) { + assert(i < write_composite_length()); + return buffer_[i + write_position_]; +} + +const uint8_t &array_buffer::write(size_t i) const { + assert(i < write_composite_length()); + return buffer_[i + write_position_]; +} +error array_buffer::write_require_length(size_t bytes) { + size_t write_remain = write_composite_length(); + if (bytes > write_remain) { + return make_error(); + } + return no_error(); +} + +} // namespace saw diff --git a/modules/core/buffer.h b/modules/core/buffer.h new file mode 100644 index 0000000..f0cf76e --- /dev/null +++ b/modules/core/buffer.h @@ -0,0 +1,199 @@ +#pragma once + +#include "error.h" + +#include +#include +#include +#include +#include +#include + +namespace saw { +/* + * Access class to reduce templated BufferSegments bloat + */ +class buffer { +protected: + ~buffer() = default; + +public: + virtual size_t read_position() const = 0; + virtual size_t read_composite_length() const = 0; + virtual size_t read_segment_length(size_t offset = 0) const = 0; + virtual void read_advance(size_t bytes) = 0; + + virtual uint8_t &read(size_t i = 0) = 0; + virtual const uint8_t &read(size_t i = 0) const = 0; + + virtual size_t write_position() const = 0; + virtual size_t write_composite_length() const = 0; + virtual size_t write_segment_length(size_t offset = 0) const = 0; + virtual void write_advance(size_t bytes) = 0; + + virtual uint8_t &write(size_t i = 0) = 0; + virtual const uint8_t &write(size_t i = 0) const = 0; + + /* + * Sometime buffers need to grow with a little more control + * than with push and pop for more efficient calls. + * There is nothing you can do if read hasn't been filled, but at + * least write can be increased if it is demanded. + */ + virtual error write_require_length(size_t bytes) = 0; + + error push(const uint8_t &value); + error push(const uint8_t &buffer, size_t size); + error pop(uint8_t &value); + error pop(uint8_t &buffer, size_t size); + +}; + +/** + * Converts a buffer to a string for convenience cases + */ +std::string convert_to_string(const buffer& buf); + +/** + * + */ + +/* + * A viewer class for buffers. + * Working on the reference buffer invalidates the buffer view + */ +class buffer_view : public buffer { +private: + buffer *buffer_; + size_t read_offset_; + size_t write_offset_; + +public: + buffer_view(buffer &); + + size_t read_position() const override; + size_t read_composite_length() const override; + size_t read_segment_length(size_t offset = 0) const override; + void read_advance(size_t bytes) override; + + uint8_t &read(size_t i = 0) override; + const uint8_t &read(size_t i = 0) const override; + + size_t write_position() const override; + size_t write_composite_length() const override; + size_t write_segment_length(size_t offset = 0) const override; + void write_advance(size_t bytes) override; + + uint8_t &write(size_t i = 0) override; + const uint8_t &write(size_t i = 0) const override; + + error write_require_length(size_t bytes) override; + + size_t read_offset() const; + size_t write_offset() const; +}; + +/* + * Buffer size meant for default allocation size of the ringbuffer since + * this class currently doesn't support proper resizing + */ +constexpr size_t RING_BUFFER_MAX_SIZE = 4096; +/* + * Buffer wrapping around if read caught up + */ +class ring_buffer final : public buffer { +private: + std::vector buffer_; + size_t read_position_; + size_t write_position_; + bool write_reached_read_ = false; + +public: + ring_buffer(); + ring_buffer(size_t size); + + inline size_t size() const { return buffer_.size(); } + + inline uint8_t &operator[](size_t i) { return buffer_[i]; } + inline const uint8_t &operator[](size_t i) const { return buffer_[i]; } + + size_t read_position() const override; + size_t read_composite_length() const override; + size_t read_segment_length(size_t offset = 0) const override; + void read_advance(size_t bytes) override; + + uint8_t &read(size_t i = 0) override; + const uint8_t &read(size_t i = 0) const override; + + size_t write_position() const override; + size_t write_composite_length() const override; + size_t write_segment_length(size_t offset = 0) const override; + void write_advance(size_t bytes) override; + + uint8_t &write(size_t i = 0) override; + const uint8_t &write(size_t i = 0) const override; + + error write_require_length(size_t bytes) override; +}; + +/* + * One time buffer + */ +class array_buffer : public buffer { +private: + std::vector buffer_; + + size_t read_position_; + size_t write_position_; + +public: + array_buffer(size_t size); + + size_t read_position() const override; + size_t read_composite_length() const override; + size_t read_segment_length(size_t offset = 0) const override; + void read_advance(size_t bytes) override; + + uint8_t &read(size_t i = 0) override; + const uint8_t &read(size_t i = 0) const override; + + size_t write_position() const override; + size_t write_composite_length() const override; + size_t write_segment_length(size_t offset = 0) const override; + void write_advance(size_t bytes) override; + + uint8_t &write(size_t i = 0) override; + const uint8_t &write(size_t i = 0) const override; + + error write_require_length(size_t bytes) override; +}; + +class chain_array_buffer : public buffer { +private: + std::deque buffer_; + + size_t read_position_; + size_t write_position_; + +public: + chain_array_buffer(); + + size_t read_position() const override; + size_t read_composite_length() const override; + size_t read_segment_length(size_t offset = 0) const override; + void read_advance(size_t bytes) override; + + uint8_t &read(size_t i = 0) override; + const uint8_t &read(size_t i = 0) const override; + + size_t write_position() const override; + size_t write_composite_length() const override; + size_t write_segment_length(size_t offset = 0) const override; + void write_advance(size_t bytes) override; + + uint8_t &write(size_t i = 0) override; + const uint8_t &write(size_t i = 0) const override; + + error write_require_length(size_t bytes) override; +}; +} // namespace saw diff --git a/modules/core/common.h b/modules/core/common.h new file mode 100644 index 0000000..a06c238 --- /dev/null +++ b/modules/core/common.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include + +namespace saw { + +#define SAW_CONCAT_(x, y) x##y +#define SAW_CONCAT(x, y) SAW_CONCAT_(x, y) +#define SAW_UNIQUE_NAME(prefix) SAW_CONCAT(prefix, __LINE__) + +#define SAW_FORBID_COPY(classname) \ + classname(const classname &) = delete; \ + classname &operator=(const classname &) = delete + +#define SAW_FORBID_MOVE(classname) \ + classname(classname &&) = delete; \ + classname &operator=(classname &&) = delete + +#define SAW_DEFAULT_COPY(classname) \ + classname(const classname &) = default; \ + classname &operator=(const classname &) = default + +#define SAW_DEFAULT_MOVE(classname) \ + classname(classname &&) = default; \ + classname &operator=(classname &&) = default + +// In case of C++20 +#define SAW_ASSERT(expression) \ + assert(expression); \ + if (!(expression)) [[unlikely]] + +template using maybe = std::optional; + +template using own = std::unique_ptr; + +template using our = std::shared_ptr; + +template using lent = std::weak_ptr; + +template own heap(Args &&...args) { + return own(new T(std::forward(args)...)); +} + +template our share(Args &&...args) { + return std::make_shared(std::forward(args)...); +} + +template T instance() noexcept; + +template struct return_type_helper { + typedef decltype(instance()(instance())) Type; +}; +template struct return_type_helper { + typedef decltype(instance()()) Type; +}; + +template +using return_type = typename return_type_helper::Type; + +struct void_t {}; + +template struct void_fix { typedef T Type; }; +template <> struct void_fix { typedef void_t Type; }; +template using fix_void = typename void_fix::Type; + +template struct void_unfix { typedef T Type; }; +template <> struct void_unfix { typedef void Type; }; +template using unfix_void = typename void_unfix::Type; + +template constexpr bool always_false = false; + +} // namespace saw diff --git a/modules/core/error.cpp b/modules/core/error.cpp new file mode 100644 index 0000000..360e628 --- /dev/null +++ b/modules/core/error.cpp @@ -0,0 +1,156 @@ +#include "error.h" + +namespace saw { +error::error(error::code code_, bool is_critical__) + : error_code_{static_cast(code_)}, is_critical_{is_critical__} {} + +error::error(error::code code_, bool is_critical__, const std::string_view &msg) + : + error_code_{static_cast(code_)} + , is_critical_{is_critical__}, error_message_{msg}{} + +error::error(error &&error) + : + error_code_{std::move(error.error_code_)} + , is_critical_{std::move(error.is_critical_)} + , error_message_{std::move(error.error_message_)}{} + +const std::string_view error::get_category() const { + auto& reg = impl::get_error_registry(); + + auto eov = reg.search_category(error_code_); + SAW_ASSERT(eov.is_value()){ + return "Error category not found. Report this error to the forstio maintainer"; + } + + return eov.get_value(); +} + +const std::string_view error::get_message() const { + return std::visit( + [this](auto &&arg) -> const std::string_view { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + return std::string_view{arg}; + } else if constexpr (std::is_same_v) { + return arg; + } else { + return "Error in class Error. Good luck :)"; + } + }, + error_message_); +} + +bool error::failed() const { + return !this->is_type(); +} + +bool error::is_critical() const { + return is_critical_; +} + +bool error::is_recoverable() const { + return !is_critical_; +} + +error error::copy_error() const { + auto copy_error_code = error_code_; + error error{copy_error_code, is_critical_}; + + try { + error.error_message_ = error_message_; + } catch (const std::bad_alloc &) { + error.error_message_ = + std::string_view{"Error while copying Error string. Out of memory"}; + } + + return error; +} + +error::code error::get_id() const { return error_code_; } + +namespace impl { +error_registry& get_error_registry() { + static own reg = nullptr; + if(!reg){ + reg = heap(); + } + + assert(reg); + return *reg; +} +} + +error no_error(){ + return make_error(); +} + +namespace impl { +error_registry::error_registry(): + infos_{ + { + err::no_error::description, + err::no_error::is_critical + }, + { + err::not_found::description, + err::not_found::is_critical + }, + { + err::out_of_memory::description, + err::out_of_memory::is_critical + } + } +{} + +error_or error_registry::search_category(const error::code& id) const { + if( id >= infos_.size()){ + return make_error(); + } + + return infos_.at(id).description; +} + +error_or error_registry::search_id(const std::string_view& desc)const{ + /** + * Search the index in the vector + */ + size_t i{}; + size_t info_max_size = std::min(infos_.size(), std::numeric_limits::max()); + for(i = 0; i < info_max_size; ++i){ + if(infos_.at(i).description == desc){ + break; + } + } + + if(i == info_max_size){ + return make_error(); + } + + return static_cast(i); +} + +error_or error_registry::search_or_register_id(const std::string_view& desc, bool is_critical){ + auto err_or_id = search_id(desc); + + if(err_or_id.is_value()){ + return err_or_id.get_value(); + } + + auto& err = err_or_id.get_error(); + + if(err.is_type()){ + size_t new_index = infos_.size(); + if(new_index == std::numeric_limits::max()){ + return make_error("Error registry ids are exhausted"); + } + infos_.emplace_back(error_info{desc, is_critical}); + return static_cast(new_index); + } + + return std::move(err); +} +} + +} // namespace saw diff --git a/modules/core/error.h b/modules/core/error.h new file mode 100644 index 0000000..e816734 --- /dev/null +++ b/modules/core/error.h @@ -0,0 +1,248 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" + +namespace saw { +/** + * Utility class for generating errors. Has a base distinction between + * critical and recoverable errors. Additional code ids can be provided to the + * constructor if additional distinctions are necessary. + */ +class error { +public: + using code = uint32_t; +private: + code error_code_; + bool is_critical_; + std::variant error_message_; + +public: + error(error::code id, bool is_critical); + error(error::code id, bool is_critical, const std::string_view &msg); + error(error &&error); + + SAW_FORBID_COPY(error); + + error &operator=(error &&) = default; + + const std::string_view get_message() const; + + const std::string_view get_category() const; + + bool failed() const; + + bool is_critical() const; + bool is_recoverable() const; + + /** + * Replaces the copy constructor. We need this since we want to explicitly copy and not implicitly + */ + error copy_error() const; + + code get_id() const; + + template + bool is_type() const; +}; + +template +class error_or; + +namespace impl { + +class error_registry { +private: + struct error_info { + error_info() = delete; + error_info(const std::string_view& d_, bool critical_):description{d_}, is_critical{critical_}{} + + std::string_view description; + bool is_critical; + }; + + std::vector infos_; +public: + error_registry(); + + SAW_FORBID_COPY(error_registry); + SAW_FORBID_MOVE(error_registry); + + error_or search_category(const error::code& id) const; + + error_or search_id(const std::string_view& desc) const; + + error_or search_or_register_id(const std::string_view& desc, bool is_critical); +}; + +error_registry& get_error_registry(); + +template +error::code get_template_id(){ + static error::code id = std::numeric_limits::max(); + + if(id == std::numeric_limits::max()){ + auto& reg = get_error_registry(); + + auto err_or_id = reg.search_or_register_id(T::description, T::is_critical); + if(err_or_id.is_error()){ + return std::numeric_limits::max(); + } + + id = err_or_id.get_value(); + } + + return id; +} +} + +template error make_error(const std::string_view& generic){ + error::code id = impl::get_template_id(); + + return error{id, T::is_critical, generic}; +} + +template error make_error(){ + error::code id = impl::get_template_id(); + + return error{id, T::is_critical}; +} + +error make_error(error::code code, const std::string_view &generic); + + +namespace err { +struct no_error { + static constexpr std::string_view description = "No error has occured"; + static constexpr bool is_critical = false; +}; + +struct recoverable { + static constexpr std::string_view description = "Generic recoverable error"; + static constexpr bool is_critical = false; +}; + +struct critical { + static constexpr std::string_view description = "Generic critical error"; + static constexpr bool is_critical = true; +}; + +struct buffer_exhausted { + static constexpr std::string_view description = "Buffer Exhausted"; + static constexpr bool is_critical = false; +}; + +struct not_found { + static constexpr std::string_view description = "Not found"; + static constexpr bool is_critical = false; +}; + +struct out_of_memory { + static constexpr std::string_view description = "Out of memory"; + static constexpr bool is_critical = true; +}; + +struct invalid_state { + static constexpr std::string_view description = "Invalid state"; + static constexpr bool is_critical = true; +}; + +struct not_supported { + static constexpr std::string_view description = "Not supported"; + static constexpr bool is_critical = false; +}; + +struct not_implemented { + static constexpr std::string_view description = "Not implemented"; + static constexpr bool is_critical = true; +}; + +struct already_exists { + static constexpr std::string_view description = "Already exists"; + static constexpr bool is_critical = false; +}; +} + +/** + * Shorthand for no error happened + */ +error no_error(); + +/** + * Exception alternative. Since I code without exceptions this class is + * essentially a kind of exception replacement. + */ +template class error_or; + +class error_or_value { +public: + virtual ~error_or_value() = default; + + template error_or> &as() { + return static_cast> &>(*this); + } + + template const error_or> &as() const { + return static_cast> &>(*this); + } +}; + +template class error_or final : public error_or_value { +private: + std::variant> value_or_error_; + + static_assert(!std::is_same_v, + "Don't use internal private types"); + +public: + error_or():value_or_error_{fix_void{}}{} + error_or(const fix_void &value) : value_or_error_{value} {} + + error_or(fix_void &&value) : value_or_error_{std::move(value)} {} + + error_or(const error &error) : value_or_error_{error} {} + error_or(error &&error) : value_or_error_{std::move(error)} {} + + bool is_value() const { + return std::holds_alternative>(value_or_error_); + } + + bool is_error() const { + return std::holds_alternative(value_or_error_); + } + + class error &get_error() { + return std::get(value_or_error_); + } + + const class error &get_error() const { + return std::get(value_or_error_); + } + + fix_void &get_value() { return std::get>(value_or_error_); } + + const fix_void &get_value() const { + return std::get>(value_or_error_); + } +}; + +template class error_or> { +private: + error_or() = delete; +}; + +template +bool error::is_type() const { + + return error_code_ == impl::get_template_id(); +} + +} // namespace saw diff --git a/modules/core/id.h b/modules/core/id.h new file mode 100644 index 0000000..d836648 --- /dev/null +++ b/modules/core/id.h @@ -0,0 +1,54 @@ +#pragma once + +namespace saw { +/** + * ID class which is tied to it's representing class + */ +template +class id { +private: + /** + * Alias for the value type representing the ID + */ + using type = uint64_t; + + /** + * The low level value + */ + type value_; +public: + /** + * Basic constructor for the id class + */ + id(type val): + value_{val} + {} + + SAW_DEFAULT_COPY(id); + SAW_DEFAULT_MOVE(id); + + /** + * Equal operator for the id. + * Returns true if equal, false otherwise. + */ + bool operator==(const id& rhs) const { + return value_ == rhs.value_; + } + + /** + * Unequal operator for the id. + * Returns false if equal, true otherwise. + */ + bool operator!=(const id& rhs) const { + return !(*this == rhs); + } + + /** + * Returns a const ref of the underlying base type. + * Mostly used for internal purposes. + */ + const type& get_value() const { + return value_; + } +}; +} diff --git a/modules/core/id_map.h b/modules/core/id_map.h new file mode 100644 index 0000000..d8329cf --- /dev/null +++ b/modules/core/id_map.h @@ -0,0 +1,137 @@ +#pragma once + +#include "id.h" +#include "error.h" + +#include + +namespace saw { +/** + * Fast random access id based container. + * + * Access - O(1) + * Insert - O(1) + * Erase - O(n) ? Dunno + */ +template +class id_map final { +private: + /** + * Container which stores the primary data + */ + std::vector data_; + /** + * Container which tracks free'd/fragmented elements within the + * main container + */ + std::deque> free_ids_; + +private: + /** + * Tries to reduce top ids + */ + void reduce_free_ids() noexcept { + for(;;){ + if(free_ids_.empty()){ + break; + } + + if((free_ids_.front().get_value() + 1) < data_.size()){ + break; + } + + /** + * Can this throw? + */ + free_ids_.pop_front(); + } + } +public: + /** + * Default constructor + */ + id_map() = default; + + SAW_FORBID_COPY(id_map); + SAW_DEFAULT_MOVE(id_map); + + /** + * Inserts an element into the container and returns either an id on success + * or an error on failure. + */ + error_or> insert(T val) noexcept { + /// @todo Fix size_t and id base type + if(free_ids_.empty()){ + try { + size_t i = data_.size(); + data_.emplace_back(std::move(val)); + return saw::id{i}; + } catch(std::exception& e) { + return make_error(); + } + } else { + auto f_id = std::move(free_ids_.back()); + free_ids_.pop_back(); + data_.at(f_id.get_value()) = std::move(val); + return f_id; + } + + exit(-1); + // Dummy return since this is not reachable + return make_error(); + } + + /** + * Erase a value at this id. + */ + error_or erase(const id& val) noexcept { + /** + * If id is bigger than the available vector then return an error. + */ + if(val.get_value() >= data_.size()){ + return make_error("ID is too large"); + } + + /** + * Check if it's the highest ID. Then we can just try to reduce the highest + * IDs. + */ + if((val.get_value() + 1) == data_.size()){ + data_.pop_back(); + this->reduce_free_ids(); + if(data_.size()*2 <= data_.capacity()){ + try { + data_.shrink_to_fit(); + }catch(std::exception& e){ + return make_error(); + } + } + return void_t{}; + } + + /** + * Check if ID already exists with the free IDs. + * This would mean that a double free has occured. + */ + auto find_id = std::find(free_ids_.begin(), free_ids_.end(), val); + if(find_id != free_ids_.end()){ + return make_error("ID value has already been freed"); + } + + /** + * Insert id into deque and sort it. + */ + try { + free_ids_.push_back(val); + } catch(std::exception& e){ + return make_error(); + } + std::stable_sort(free_ids_.begin(), free_ids_.end(), + [](const id& left, const id& right) -> bool { + return left.get_value() > right.get_value(); + } + ); + return void_t{}; + } +}; +} diff --git a/modules/core/mcts.h b/modules/core/mcts.h new file mode 100644 index 0000000..8a8f5ea --- /dev/null +++ b/modules/core/mcts.h @@ -0,0 +1,52 @@ +#pragma once + +#include "tree.h" + +namespace saw { +template +class mcts_tree { +private: + struct value { + uint64_t numerater; + uint64_t denominater; + T state; + + value() = default; + value(T st): + numerater{0}, + denominater{0}, + state{std::move(st)} + {} + }; + + tree_container> data_; +public: + mcts_tree() = default{ + data_.add(value{}); + } + + mcts_tree(T state){ + data_.add(value{std::move(state)}); + } + + size_t size() const { + return data_.size() - 1; + } + + T& get_state(){ + return data_.at(0).get_value().state; + } + + const T& get_state() const { + return data_.at(0).get_value().state; + } + + mcts_tree& get_tree(size_t i){ + return data_.at(i+1).get_tree(); + } + + const mcts_tree& get_tree(size_t i) const { + return data_.at(i+1).get_tree(); + } +}; +} diff --git a/modules/core/platonic.h b/modules/core/platonic.h new file mode 100644 index 0000000..eefe99f --- /dev/null +++ b/modules/core/platonic.h @@ -0,0 +1,103 @@ +#pragma once + +#include "error.h" + +namespace saw { +namespace impl { +/** + * + */ +template +struct platonic_helper { + static_assert(always_false, "Unsupported platonic body. Alternatively it's not a platonic body"); +}; + +template +struct platonic_helper { + static constexpr surface_edges = 3u; +/* + static constexpr std::array, 4u> normals = { + {0.0, 0.0, -1.0}, // 1 + {}, // 2 + {}, // 3 + {} // 4 + }; +*/ +}; + +template +struct platonic_helper { + static constexpr surface_edges = 4u; + + static constexpr std::array, 6u> normals = { + { 1.0, 0.0, 0.0}, // 1 + {-1.0, 0.0, 0.0}, // 2 + { 0.0, 1.0, 0.0}, // 3 + { 0.0,-1.0, 0.0}, // 4 + { 0.0, 0.0, 1.0}, // 5 + { 0.0, 0.0,-1.0} // 6 + }; +}; + +template +struct platonic_helper { + static constexpr uint8_t surface_edges = 3u; +/* + static constexpr std::array, 20u> normals = { + {}, // 1 + {}, // 2 + {}, // 3 + {}, // 4 + {}, // 5 + {}, // 6 + {}, // 7 + {}, // 8 + {}, // 9 + {}, // 10 + {}, // 11 + {}, // 12 + {}, // 13 + {}, // 14 + {}, // 15 + {}, // 16 + {}, // 17 + {}, // 18 + {}, // 19 + {} // 20 + }; +*/ +}; +} +/** + * Container for describing each platonic body with + * helpers describing the orientation of each body. + */ +template +class platonic { +private: + /** + * Storage for the surfaces + */ + std::array surfaces_; +public: + constexpr uint8_t get_surface_edge_size() constexpr { + return platonic_helper::surface_edges; + } + + constexpr uint8_t get_surface_size() constexpr { + return N; + } + + T& at(uint8_t i){ + return surface_.at(i); + } + + const T& at(uint8_t i) const { + return surface_.at(i); + } + + constexpr std::array& get_surface_normal(size_t i) constexpr { + + } +}; +} diff --git a/modules/core/string_literal.h b/modules/core/string_literal.h new file mode 100644 index 0000000..ccc8f49 --- /dev/null +++ b/modules/core/string_literal.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +namespace saw { +/** + * Helper object which creates a templated string from the provided string + * literal. It guarantees compile time uniqueness and thus allows using strings + * in template parameters. + */ +template class string_literal { +public: + static_assert(N > 0, "string_literal needs a null terminator"); + + constexpr string_literal(const CharT (&input)[N]) noexcept { + for (size_t i = 0; i < N; ++i) { + data[i] = input[i]; + } + } + + std::array data{}; + + constexpr std::string_view view() const noexcept { + return std::string_view{data.data()}; + } + + constexpr bool + operator==(const string_literal &) const noexcept = default; + + template + constexpr bool + operator==(const string_literal &) const noexcept { + return false; + } + + template + constexpr string_literal operator+(const string_literal& rhs) const noexcept { + CharT sum[N+NR-1]; + + // The weird i+1 happens due to needing to skip the '\0' terminator + for(size_t i = 0; (i+1) < N; ++i){ + sum[i] = data[i]; + } + for(size_t i = 0; i < NR; ++i){ + sum[i+N-1] = rhs.data[i]; + } + + return string_literal{sum}; + } +}; + +template +constexpr string_literal operator""_sl() { + return string_literal{{Chars..., '\0'}}; +} +} // namespace saw diff --git a/modules/core/templates.h b/modules/core/templates.h new file mode 100644 index 0000000..2eb0f7e --- /dev/null +++ b/modules/core/templates.h @@ -0,0 +1,150 @@ +#pragma once + +#include "string_literal.h" + +namespace saw { + +template struct parameter_pack_index; + +template struct parameter_pack_index { + static constexpr size_t value = 0u; +}; + +template +struct parameter_pack_index { + static constexpr size_t value = + 1u + parameter_pack_index::value; +}; + +template struct parameter_pack_type { + static_assert(always_false, "Should've been caught by the specializations"); +}; + +template struct parameter_pack_type<0, TN, T...> { + using type = TN; +}; + +template +struct parameter_pack_type { + static_assert(sizeof...(T) > 0, "Exhausted parameters"); + static_assert(N > 0, "Invalid number. Should've been caught"); + using type = typename parameter_pack_type::type; +}; +/* + * Nightmare inducing compiler problems found here. Somehow non-type + * string_literals cannot be resolved as non-type primitive template values can. + * This is the workaround + */ +template +struct parameter_key_pack_index_helper { + static constexpr size_t value = + (V == Key0) + ? (0u) + : (1u + parameter_key_pack_index_helper::value); +}; + +template +struct parameter_key_pack_index_helper { + static constexpr size_t value = (V == Key0) ? (0u) : (1u); +}; + +template +struct parameter_key_pack_index { + static constexpr size_t value = + parameter_key_pack_index_helper::value; + static_assert(value < sizeof...(Keys), + "Provided string_literal doesn't exist in searched list"); +}; + +template +struct parameter_key_pack_type_helper { + static constexpr string_literal literal = parameter_key_pack_type_helper::literal; +}; + +template +struct parameter_key_pack_type_helper { + static constexpr string_literal literal = Key0; +}; + +template +struct parameter_key_pack_type { + static constexpr string_literal literal = parameter_key_pack_type_helper::literal; + + static_assert(i < sizeof...(Keys), "Provided index is too large for list"); +}; + +template +struct parameter_pack_value_helper { + static constexpr T value = parameter_pack_value_helper::value; +}; + +template +struct parameter_pack_value_helper { + static constexpr T value = V0; +}; + +template +struct parameter_pack_value { + static constexpr T value = parameter_pack_value_helper::value; + static_assert(i < sizeof...(Values), "Provided index is too large for list"); +}; + +template +struct ct_multiply; + +template +struct ct_multiply { + static constexpr T value = 1; +}; + +template +struct ct_multiply { + static constexpr T value = V0 * ct_multiply::value; +}; + +namespace impl { +template +struct ct_convert_digits_table_helper { + static_assert(i <= 15, "Only conversion up to hex is supported"); + + static constexpr std::array table = { + '0', '1', '2', '3', + '4', '5', '6', '7', + '8', '9', 'A', 'B', + 'C', 'D', 'E', 'F' + }; + + static constexpr T value = table[i]; +}; + +template +struct ct_convert_digits_helper { + static constexpr size_t size = ct_convert_digits_helper::size; + static constexpr std::array value = ct_convert_digits_helper::value; + static constexpr string_literal literal = ct_convert_digits_helper::literal; +}; + +template +struct ct_convert_digits_helper<0, Base, Digs...> { + static constexpr size_t size = sizeof...(Digs); + static constexpr std::array value = {Digs...}; + static constexpr string_literal literal = {{ct_convert_digits_table_helper::value..., '\0'}}; +}; + +template +struct ct_convert_digits_helper<0, Base> { + static constexpr size_t size = 0; + static constexpr std::array value = {0}; + static constexpr string_literal literal = "0"_sl; +}; +} + +template +struct ct_convert_to_digits { + static_assert(Base <= 16, "Only conversion up to hex is supported"); + + static constexpr size_t size = impl::ct_convert_digits_helper::size; + static constexpr std::array value = impl::ct_convert_digits_helper::value; + static constexpr string_literal literal = impl::ct_convert_digits_helper::literal; +}; +} diff --git a/modules/core/tree.h b/modules/core/tree.h new file mode 100644 index 0000000..68fa20a --- /dev/null +++ b/modules/core/tree.h @@ -0,0 +1,248 @@ +#pragma once + +#include +#include + +#include "error.h" + +namespace saw { +/** + * Container with a simplistic approach to a branch + */ +template +class branch; + +/** + * Tree object holding branches. + * + * The name comes from the fact a tree is acting as a node while the branch class is the + * edge to a leaf or other nodes. A tree holds the branches while the branch either has + * a leaf or another sub tree. + */ +template +class tree_container final { +private: + /** + * Object holding the treeed branch instances + */ + std::vector> children_; +public: + /** + * Default constructor + */ + tree_container() = default; + + /** + * Destructor + */ + ~tree_container() = default; + + SAW_FORBID_COPY(tree_container); + SAW_DEFAULT_MOVE(tree_container); + + /** + * Reserve space for siz elements + */ + error_or reserve(std::size_t siz){ + try{ + children_.reserve(siz); + }catch(const std::exception& e){ + (void) e; + + return make_error(); + } + + return void_t{}; + } + + /** + * Add a branch with a leaf attached to the tree + */ + error_or add(T leaf) { + std::size_t index = size(); + try { + /** + * Technically we're adding a leaf on a branch + */ + children_.emplace_back(std::move(leaf)); + }catch(const std::exception& e){ + (void)e; + + return make_error(); + } + + return index; + } + + /** + * Add a branch to the tree with a tree attached + */ + error_or add() { + std::size_t index = size(); + try { + + children_.emplace_back(Tree{}); + }catch(const std::exception& e){ + (void)e; + + return make_error(); + } + + return index; + } + + /** + * Returns the amount of branches contained within this tree level + */ + std::size_t size() const { + return children_.size(); + } + + /** + * Returns the branch at i + */ + branch& at(std::size_t i){ + return children_.at(i); + } + + /** + * Returns the branch at i + */ + const branch& at(std::size_t i) const { + return children_.at(i); + } +}; + +template +class branch final { +private: + using type = std::variant; + type tov_; + + /** + * We're friend classing the tree since it's way easier this way and the branch and tree + * class are intertwined heavily anyway. + */ +public: + /** + * + */ + branch():tov_{Tree{}}{} + + branch(Tree nd):tov_{std::move(nd)}{} + + branch(T val):tov_{std::move(val)}{} + + SAW_FORBID_COPY(branch); + SAW_DEFAULT_MOVE(branch); + + template + bool is() const { + return std::holds_alternative(tov_); + } + + bool is_tree() const { + return std::holds_alternative(tov_); + } + + bool is_value() const { + return std::holds_alternative(tov_); + } + + template + NT& get() { + return std::get(tov_); + } + + template + const NT& get() const { + return std::get(tov_); + } + + Tree& get_tree(){ + return std::get(tov_); + } + + const Tree& get_tree() const { + return std::get(tov_); + } + + T& get_value(){ + return std::get(tov_); + } + + const T& get_value() const { + return std::get(tov_); + } + + template + error_or extract(){ + if(!is()){ + return make_error(); + } + + NT nd = std::move(std::get(tov_)); + tov_ = Tree{}; + + return nd; + } + + template + error_or replace(type nd){ + auto eon = extract(); + if(eon.is_error()){ + return eon; + } + + tov_ = std::move(nd); + + return eon; + } + + error_or extract_tree() { + return extract(); + } + + error_or replace_tree(type nd){ + return replace(std::move(nd)); + } + + error_or extract_value() { + return extract(); + } + + error_or replace_value(type nd){ + return replace(std::move(nd)); + } +}; + +template +class tree { + private: + tree_container> data_; + public: + error_or reserve(std::size_t size){ + return data_.reserve(size); + } + + size_t size() const { + return data_.size(); + } + + error_or add() { + return data_.add(); + } + + error_or add(T leaf){ + return data_.add(std::move(leaf)); + } + + branch>& at(size_t i){ + return data_.at(i); + } + + const branch>& at(size_t i) const { + return data_.at(i); + } +}; +} diff --git a/modules/device-hip/.nix/derivation.nix b/modules/device-hip/.nix/derivation.nix new file mode 100644 index 0000000..6849e6e --- /dev/null +++ b/modules/device-hip/.nix/derivation.nix @@ -0,0 +1,30 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-io_codec"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/device-hip/SConscript b/modules/device-hip/SConscript new file mode 100644 index 0000000..4ab02d6 --- /dev/null +++ b/modules/device-hip/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +dev_hip_env = env.Clone(); + +dev_hip_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +dev_hip_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += dev_hip_env.sources; +env.headers += dev_hip_env.headers; + +## Shared lib +objects_shared = [] +dev_hip_env.add_source_files(objects_shared, dev_hip_env.sources, shared=True); +dev_hip_env.library_shared = dev_hip_env.SharedLibrary('#build/forstio-device-hip', [objects_shared]); + +## Static lib +objects_static = [] +dev_hip_env.add_source_files(objects_static, dev_hip_env.sources, shared=False); +dev_hip_env.library_static = dev_hip_env.StaticLibrary('#build/forstio-device-hip', [objects_static]); + +# Set Alias +env.Alias('library_device-hip', [dev_hip_env.library_shared, dev_hip_env.library_static]); + +env.targets += ['library_device-hip']; + +# Install +env.Install('$prefix/lib/', [dev_hip_env.library_shared, dev_hip_env.library_static]); +env.Install('$prefix/include/forstio/device/hip/', [dev_hip_env.headers]); diff --git a/modules/device-hip/SConstruct b/modules/device-hip/SConstruct new file mode 100644 index 0000000..4e6e150 --- /dev/null +++ b/modules/device-hip/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-io']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/device-hip/rpc.h b/modules/device-hip/rpc.h new file mode 100644 index 0000000..b6421ba --- /dev/null +++ b/modules/device-hip/rpc.h @@ -0,0 +1,7 @@ +#pragma once + +namespace saw { +namespace rmt { +struct DeviceHip {}; +} +} diff --git a/modules/io-fs/.nix/derivation.nix b/modules/io-fs/.nix/derivation.nix new file mode 100644 index 0000000..c8a4962 --- /dev/null +++ b/modules/io-fs/.nix/derivation.nix @@ -0,0 +1,33 @@ +{ lib +, stdenvNoCC +, scons +, clang +, clang-tools +, version +, forstio +}: + +let + +in stdenvNoCC.mkDerivation { + pname = "forstio-io-fs"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + forstio.io + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/io-fs/SConscript b/modules/io-fs/SConscript new file mode 100644 index 0000000..0d85f62 --- /dev/null +++ b/modules/io-fs/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +io_env = env.Clone(); + +io_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +io_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += io_env.sources; +env.headers += io_env.headers; + +## Shared lib +objects_shared = [] +io_env.add_source_files(objects_shared, io_env.sources, shared=True); +io_env.library_shared = io_env.SharedLibrary('#build/forstio-io-fs', [objects_shared]); + +## Static lib +objects_static = [] +io_env.add_source_files(objects_static, io_env.sources, shared=False); +io_env.library_static = io_env.StaticLibrary('#build/forstio-io-fs', [objects_static]); + +# Set Alias +env.Alias('library_io', [io_env.library_shared, io_env.library_static]); + +env.targets += ['library_io']; + +# Install +env.Install('$prefix/lib/', [io_env.library_shared, io_env.library_static]); +env.Install('$prefix/include/forstio/io/fs/', [io_env.headers]); diff --git a/modules/io-fs/SConstruct b/modules/io-fs/SConstruct new file mode 100644 index 0000000..4cccf82 --- /dev/null +++ b/modules/io-fs/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-async']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/io-tls/.nix/derivation.nix b/modules/io-tls/.nix/derivation.nix new file mode 100644 index 0000000..23d8ba6 --- /dev/null +++ b/modules/io-tls/.nix/derivation.nix @@ -0,0 +1,32 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +, gnutls +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-io-tls"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.io + gnutls + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/io-tls/SConscript b/modules/io-tls/SConscript new file mode 100644 index 0000000..4f88f37 --- /dev/null +++ b/modules/io-tls/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +io_tls_env = env.Clone(); + +io_tls_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +io_tls_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += io_tls_env.sources; +env.headers += io_tls_env.headers; + +## Shared lib +objects_shared = [] +io_tls_env.add_source_files(objects_shared, io_tls_env.sources, shared=True); +io_tls_env.library_shared = io_tls_env.SharedLibrary('#build/forstio-io-tls', [objects_shared]); + +## Static lib +objects_static = [] +io_tls_env.add_source_files(objects_static, io_tls_env.sources, shared=False); +io_tls_env.library_static = io_tls_env.StaticLibrary('#build/forstio-io-tls', [objects_static]); + +# Set Alias +env.Alias('library_io_tls', [io_tls_env.library_shared, io_tls_env.library_static]); + +env.targets += ['library_io_tls']; + +# Install +env.Install('$prefix/lib/', [io_tls_env.library_shared, io_tls_env.library_static]); +env.Install('$prefix/include/forstio/io/tls/', [io_tls_env.headers]); diff --git a/modules/io-tls/SConstruct b/modules/io-tls/SConstruct new file mode 100644 index 0000000..fbd8657 --- /dev/null +++ b/modules/io-tls/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['gnutls','forstio-io']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/io-tls/tls.cpp b/modules/io-tls/tls.cpp new file mode 100644 index 0000000..9fa143c --- /dev/null +++ b/modules/io-tls/tls.cpp @@ -0,0 +1,252 @@ +#include "tls.h" + +#include +#include + +#include + +#include + +#include + +namespace saw { + +class tls::impl { +public: + gnutls_certificate_credentials_t xcred; + +public: + impl() { + gnutls_global_init(); + gnutls_certificate_allocate_credentials(&xcred); + gnutls_certificate_set_x509_system_trust(xcred); + } + + ~impl() { + gnutls_certificate_free_credentials(xcred); + gnutls_global_deinit(); + } +}; + +static ssize_t forst_tls_push_func(gnutls_transport_ptr_t p, const void *data, + size_t size); +static ssize_t forst_tls_pull_func(gnutls_transport_ptr_t p, void *data, size_t size); + +tls::tls() : impl_{heap()} {} + +tls::~tls() {} + +tls::impl &tls::get_impl() { return *impl_; } + +class tls_io_stream final : public io_stream { +private: + own internal; + gnutls_session_t session_handle; + +public: + tls_io_stream(own internal_) : internal{std::move(internal_)} {} + + ~tls_io_stream() { gnutls_bye(session_handle, GNUTLS_SHUT_RDWR); } + + error_or read(void *buffer, size_t length) override { + ssize_t size = gnutls_record_recv(session_handle, buffer, length); + if (size < 0) { + if(gnutls_error_is_fatal(size) == 0){ + return make_error("Recoverable error on read in gnutls. TODO better error msg handling"); + // Leaving proper message handling done in previous error framework + //return recoverable_error([size](){return std::string{"Read recoverable Error "}+std::string{gnutls_strerror(size)};}, "Error read r"); + }else{ + return make_error("Fatal error on read in gnutls. TODO better error msg handling"); + } + }else if(size == 0){ + return make_error(); + } + + return static_cast(length); + } + + conveyor read_ready() override { return internal->read_ready(); } + + conveyor on_read_disconnected() override { + return internal->on_read_disconnected(); + } + + error_or write(const void *buffer, size_t length) override { + ssize_t size = gnutls_record_send(session_handle, buffer, length); + if(size < 0){ + if(gnutls_error_is_fatal(size) == 0){ + return make_error("Recoverable error on write in gnutls. TODO better error msg handling"); + }else{ + return make_error("Fatal error on write in gnutls. TODO better error msg handling"); + } + } + + return static_cast(size); + } + + conveyor write_ready() override { return internal->write_ready(); } + + gnutls_session_t &session() { return session_handle; } +}; + +tls_server::tls_server(own srv) : internal{std::move(srv)} {} + +conveyor> tls_server::accept() { + SAW_ASSERT(internal) { return conveyor>{fix_void>{nullptr}}; } + return internal->accept().then([](own stream) -> own { + /// @todo handshake + + + return heap(std::move(stream)); + }); +} + +namespace { +/* +* Small helper for setting up the nonblocking connection handshake +*/ +struct tls_client_stream_helper { +public: + own>> feeder; + conveyor_sink connection_sink; + conveyor_sink stream_reader; + conveyor_sink stream_writer; + + own stream = nullptr; +public: + tls_client_stream_helper(own>> f): + feeder{std::move(f)} + {} + + void setupTurn(){ + SAW_ASSERT(stream){ + return; + } + + stream_reader = stream->read_ready().then([this](){ + turn(); + }).sink(); + + stream_writer = stream->write_ready().then([this](){ + turn(); + }).sink(); + } + + void turn(){ + if(stream){ + // Guarantee that the receiving end is already setup + SAW_ASSERT(feeder){ + return; + } + + auto &session = stream->session(); + + int ret; + do { + ret = gnutls_handshake(session); + } while ( (ret == GNUTLS_E_AGAIN || ret == GNUTLS_E_INTERRUPTED) && gnutls_error_is_fatal(ret) == 0); + + if(gnutls_error_is_fatal(ret)){ + feeder->fail(make_error("Couldn't create Tls connection")); + stream = nullptr; + }else if(ret == GNUTLS_E_SUCCESS){ + feeder->feed(std::move(stream)); + } + } + } +}; +} + +own tls_network::listen(network_address& address) { + return heap(internal.listen(address)); +} + +conveyor> tls_network::connect(network_address& address) { + // Helper setups + auto caf = new_conveyor_and_feeder>(); + own helper = heap(std::move(caf.feeder)); + tls_client_stream_helper* hlp_ptr = helper.get(); + + // Conveyor entangled structure + auto prim_conv = internal.connect(address).then([this, hlp_ptr, addr = address.address()]( + own stream) -> error_or { + io_stream* inner_stream = stream.get(); + auto tls_stream = heap(std::move(stream)); + + auto &session = tls_stream->session(); + + gnutls_init(&session, GNUTLS_CLIENT); + + gnutls_server_name_set(session, GNUTLS_NAME_DNS, addr.c_str(), + addr.size()); + + gnutls_set_default_priority(session); + gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, + tls_.get_impl().xcred); + gnutls_session_set_verify_cert(session, addr.c_str(), 0); + + gnutls_transport_set_ptr(session, reinterpret_cast(inner_stream)); + gnutls_transport_set_push_function(session, forst_tls_push_func); + gnutls_transport_set_pull_function(session, forst_tls_pull_func); + + // gnutls_handshake_set_timeout(session, GNUTLS_DEFAULT_HANDSHAKE_TIMEOUT); + + hlp_ptr->stream = std::move(tls_stream); + hlp_ptr->setupTurn(); + hlp_ptr->turn(); + + return void_t{}; + }); + + helper->connection_sink = prim_conv.sink(); + + return caf.conveyor.attach(std::move(helper)); +} + +own tls_network::datagram(network_address& address){ + ///@unimplemented + return nullptr; +} + +static ssize_t forst_tls_push_func(gnutls_transport_ptr_t p, const void *data, + size_t size) { + io_stream *stream = reinterpret_cast(p); + if (!stream) { + return -1; + } + + error_or length = stream->write(data, size); + if (length.is_error() || !length.is_value()) { + return -1; + } + + return static_cast(length.get_value()); +} + +static ssize_t forst_tls_pull_func(gnutls_transport_ptr_t p, void *data, size_t size) { + io_stream *stream = reinterpret_cast(p); + if (!stream) { + return -1; + } + + error_or length = stream->read(data, size); + if (length.is_error() || !length.is_value()) { + return -1; + } + + return static_cast(length.get_value()); +} + +tls_network::tls_network(tls& tls_, network &network) : tls_{tls_},internal{network} {} + +conveyor> tls_network::resolve_address(const std::string &addr, + uint16_t port) { + /// @todo tls server name needed. Check validity. Won't matter later on, because gnutls should fail anyway. But + /// it's better to find the error source sooner rather than later + return internal.resolve_address(addr, port); +} + +std::optional> setup_tls_network(network &network) { + return std::nullopt; +} +} // namespace saw diff --git a/modules/io-tls/tls.h b/modules/io-tls/tls.h new file mode 100644 index 0000000..74b39ff --- /dev/null +++ b/modules/io-tls/tls.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include + +#include +#include + +namespace saw { +class tls; + +class tls_server final : public server { +private: + own internal; + +public: + tls_server(own srv); + + conveyor> accept() override; +}; + +class tls_network final : public network { +private: + tls& tls_; + network &internal; +public: + tls_network(tls& tls_, network &network_); + + conveyor> resolve_address(const std::string &addr, uint16_t port = 0) override; + + own listen(network_address& address) override; + + conveyor> connect(network_address& address) override; + + own datagram(network_address& address) override; +}; + +/** +* tls context class. +* Provides tls network class which ensures the usage of tls encrypted connections +*/ +class tls { +private: + class impl; + own impl_; +public: + tls(); + ~tls(); + + struct version { + struct tls_1_0{}; + struct tls_1_1{}; + struct tls_1_2{}; + }; + + struct options { + public: + version version; + }; + + impl &get_impl(); +private: + options options_; +}; + +std::optional> setup_tls_network(network &network); + +} // namespace saw diff --git a/modules/io/.nix/derivation.nix b/modules/io/.nix/derivation.nix new file mode 100644 index 0000000..a14bd34 --- /dev/null +++ b/modules/io/.nix/derivation.nix @@ -0,0 +1,29 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-io"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/io/SConscript b/modules/io/SConscript new file mode 100644 index 0000000..62ad58a --- /dev/null +++ b/modules/io/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +io_env = env.Clone(); + +io_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +io_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += io_env.sources; +env.headers += io_env.headers; + +## Shared lib +objects_shared = [] +io_env.add_source_files(objects_shared, io_env.sources, shared=True); +io_env.library_shared = io_env.SharedLibrary('#build/forstio-io', [objects_shared]); + +## Static lib +objects_static = [] +io_env.add_source_files(objects_static, io_env.sources, shared=False); +io_env.library_static = io_env.StaticLibrary('#build/forstio-io', [objects_static]); + +# Set Alias +env.Alias('library_io', [io_env.library_shared, io_env.library_static]); + +env.targets += ['library_io']; + +# Install +env.Install('$prefix/lib/', [io_env.library_shared, io_env.library_static]); +env.Install('$prefix/include/forstio/io/', [io_env.headers]); diff --git a/modules/io/SConstruct b/modules/io/SConstruct new file mode 100644 index 0000000..4cccf82 --- /dev/null +++ b/modules/io/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-async']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/io/io.cpp b/modules/io/io.cpp new file mode 100644 index 0000000..f0705d2 --- /dev/null +++ b/modules/io/io.cpp @@ -0,0 +1,70 @@ +#include "io.h" + +#include + +namespace saw { + +async_io_stream::async_io_stream(own str) + : stream_{std::move(str)}, + read_ready_{stream_->read_ready() + .then([this]() { read_stepper_.read_step(*stream_); }) + .sink()}, + write_ready_{stream_->write_ready() + .then([this]() { write_stepper_.write_step(*stream_); }) + .sink()}, + read_disconnected_{stream_->on_read_disconnected() + .then([this]() { + if (read_stepper_.on_read_disconnect) { + read_stepper_.on_read_disconnect->feed(); + } + }) + .sink()} {} + +void async_io_stream::read(void *buffer, size_t min_length, size_t max_length) { + SAW_ASSERT(buffer && max_length >= min_length && min_length > 0) { return; } + + SAW_ASSERT(!read_stepper_.read_task.has_value()) { return; } + + read_stepper_.read_task = read_task_and_step_helper::read_io_task{ + buffer, min_length, max_length, 0}; + read_stepper_.read_step(*stream_); +} + +conveyor async_io_stream::read_done() { + auto caf = new_conveyor_and_feeder(); + read_stepper_.read_done = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +conveyor async_io_stream::on_read_disconnected() { + auto caf = new_conveyor_and_feeder(); + read_stepper_.on_read_disconnect = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +void async_io_stream::write(const void *buffer, size_t length) { + SAW_ASSERT(buffer && length > 0) { return; } + + SAW_ASSERT(!write_stepper_.write_task.has_value()) { return; } + + write_stepper_.write_task = + write_task_and_step_helper::write_io_task{buffer, length, 0}; + write_stepper_.write_step(*stream_); +} + +conveyor async_io_stream::write_done() { + auto caf = new_conveyor_and_feeder(); + write_stepper_.write_done = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +string_network_address::string_network_address(const std::string &address, + uint16_t port) + : address_value_{address}, port_value_{port} {} + +const std::string &string_network_address::address() const { + return address_value_; +} + +uint16_t string_network_address::port() const { return port_value_; } +} // namespace saw diff --git a/modules/io/io.h b/modules/io/io.h new file mode 100644 index 0000000..7653ace --- /dev/null +++ b/modules/io/io.h @@ -0,0 +1,219 @@ +#pragma once + +#include +#include +#include "io_helpers.h" + +#include +#include + +namespace saw { +/** + * Set of error common in io + */ +namespace err { +struct disconnected { + static constexpr std::string_view description = "Disconnected"; + static constexpr bool is_critical = true; +}; + +struct resource_busy { + static constexpr std::string_view description = "Resource busy"; + static constexpr bool is_critical = false; +}; +} +/* + * Input stream + */ +class input_stream { +public: + virtual ~input_stream() = default; + + virtual error_or read(void *buffer, size_t length) = 0; + + virtual conveyor read_ready() = 0; + + virtual conveyor on_read_disconnected() = 0; +}; + +/* + * Output stream + */ +class output_stream { +public: + virtual ~output_stream() = default; + + virtual error_or write(const void *buffer, size_t length) = 0; + + virtual conveyor write_ready() = 0; +}; + +/* + * Io stream + */ +class io_stream : public input_stream, public output_stream { +public: + virtual ~io_stream() = default; +}; + +class async_input_stream { +public: + virtual ~async_input_stream() = default; + + virtual void read(void *buffer, size_t min_length, size_t max_length) = 0; + + virtual conveyor read_done() = 0; + virtual conveyor on_read_disconnected() = 0; +}; + +class async_output_stream { +public: + virtual ~async_output_stream() = default; + + virtual void write(const void *buffer, size_t length) = 0; + + virtual conveyor write_done() = 0; +}; + +class async_io_stream final : public async_input_stream, + public async_output_stream { +private: + own stream_; + + conveyor_sink read_ready_; + conveyor_sink write_ready_; + conveyor_sink read_disconnected_; + + read_task_and_step_helper read_stepper_; + write_task_and_step_helper write_stepper_; + +public: + async_io_stream(own str); + + SAW_FORBID_COPY(async_io_stream); + SAW_FORBID_MOVE(async_io_stream); + + void read(void *buffer, size_t length, size_t max_length) override; + + conveyor read_done() override; + + conveyor on_read_disconnected() override; + + void write(const void *buffer, size_t length) override; + + conveyor write_done() override; +}; + +class server { +public: + virtual ~server() = default; + + virtual conveyor> accept() = 0; +}; + +class network_address; +/** + * Datagram class. Bound to a local address it is able to receive inbound + * datagram messages and send them as well as long as an address is provided as + * well + */ +class datagram { +public: + virtual ~datagram() = default; + + virtual error_or read(void *buffer, size_t length) = 0; + virtual conveyor read_ready() = 0; + + virtual error_or write(const void *buffer, size_t length, + network_address &dest) = 0; + virtual conveyor write_ready() = 0; +}; + +class os_network_address; +class string_network_address; + +class network_address { +public: + using child_variant = + std::variant; + + virtual ~network_address() = default; + + virtual network_address::child_variant representation() = 0; + + virtual const std::string &address() const = 0; + virtual uint16_t port() const = 0; +}; + +class os_network_address : public network_address { +public: + virtual ~os_network_address() = default; + + network_address::child_variant representation() override { return this; } +}; + +class string_network_address final : public network_address { +private: + std::string address_value_; + uint16_t port_value_; + +public: + string_network_address(const std::string &address, uint16_t port); + + const std::string &address() const override; + uint16_t port() const override; + + network_address::child_variant representation() override { return this; } +}; + +class network { +public: + virtual ~network() = default; + + /** + * Resolve the provided string and uint16 to the preferred storage method + */ + virtual conveyor> + resolve_address(const std::string &addr, uint16_t port_hint = 0) = 0; + + /** + * Parse the provided string and uint16 to the preferred storage method + * Since no dns request is made here, no async conveyors have to be used. + */ + /// @todo implement + // virtual Own parseAddress(const std::string& addr, + // uint16_t port_hint = 0) = 0; + + /** + * Set up a listener on this address + */ + virtual own listen(network_address &bind_addr) = 0; + + /** + * Connect to a remote address + */ + virtual conveyor> connect(network_address &address) = 0; + + /** + * Bind a datagram socket at this address. + */ + virtual own datagram(network_address &address) = 0; +}; + +class io_provider { +public: + virtual ~io_provider() = default; + + virtual own wrap_input_fd(int fd) = 0; + + virtual network &get_network() = 0; +}; + +struct async_io_context { + own io; + event_loop &event_loop; + event_port &event_port; +}; + +error_or setup_async_io(); +} // namespace saw diff --git a/modules/io/io_helpers.cpp b/modules/io/io_helpers.cpp new file mode 100644 index 0000000..c2cf2be --- /dev/null +++ b/modules/io/io_helpers.cpp @@ -0,0 +1,85 @@ +#include "io_helpers.h" + +#include "io.h" + +#include + +namespace saw { +void read_task_and_step_helper::read_step(input_stream &reader) { + while (read_task.has_value()) { + read_io_task &task = *read_task; + + error_or n_err = reader.read(task.buffer, task.max_length); + if (n_err.is_error()) { + const error &error = n_err.get_error(); + if (error.is_critical()) { + if (read_done) { + read_done->fail(error.copy_error()); + } + read_task = std::nullopt; + } + + break; + } else if (n_err.is_value()) { + size_t n = n_err.get_value(); + if (static_cast(n) >= task.min_length && + static_cast(n) <= task.max_length) { + if (read_done) { + read_done->feed(n + task.already_read); + } + read_task = std::nullopt; + } else { + task.buffer = static_cast(task.buffer) + n; + task.min_length -= static_cast(n); + task.max_length -= static_cast(n); + task.already_read += n; + } + + } else { + if (read_done) { + read_done->fail(make_error("Read failed")); + } + read_task = std::nullopt; + } + } +} + +void write_task_and_step_helper::write_step(output_stream &writer) { + while (write_task.has_value()) { + write_io_task &task = *write_task; + + error_or n_err = writer.write(task.buffer, task.length); + + if (n_err.is_value()) { + + size_t n = n_err.get_value(); + assert(n <= task.length); + if (n == task.length) { + if (write_done) { + write_done->feed(n + task.already_written); + } + write_task = std::nullopt; + } else { + task.buffer = static_cast(task.buffer) + n; + task.length -= n; + task.already_written += n; + } + } else if (n_err.is_error()) { + const error &error = n_err.get_error(); + if (error.is_critical()) { + if (write_done) { + write_done->fail(error.copy_error()); + } + write_task = std::nullopt; + } + break; + } else { + if (write_done) { + write_done->fail(make_error("Write failed")); + } + write_task = std::nullopt; + } + } +} + +} // namespace saw diff --git a/modules/io/io_helpers.h b/modules/io/io_helpers.h new file mode 100644 index 0000000..94e37f4 --- /dev/null +++ b/modules/io/io_helpers.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +#include +#include + +namespace saw { +/* + * Helper classes for the specific driver implementations + */ + +/* + * Since I don't want to repeat these implementations for tls on unix systems + * and gnutls doesn't let me write or read into buffers I have to have this kind + * of strange abstraction. This may also be reusable for windows/macOS though. + */ +class input_stream; + +class read_task_and_step_helper { +public: + struct read_io_task { + void *buffer; + size_t min_length; + size_t max_length; + size_t already_read = 0; + }; + std::optional read_task; + own> read_done = nullptr; + + own> on_read_disconnect = nullptr; + +public: + void read_step(input_stream &reader); +}; + +class output_stream; + +class write_task_and_step_helper { +public: + struct write_io_task { + const void *buffer; + size_t length; + size_t already_written = 0; + }; + std::optional write_task; + own> write_done = nullptr; + +public: + void write_step(output_stream &writer); +}; +} // namespace saw diff --git a/modules/io/io_unix.cpp b/modules/io/io_unix.cpp new file mode 100644 index 0000000..c3b4f17 --- /dev/null +++ b/modules/io/io_unix.cpp @@ -0,0 +1,894 @@ +#ifdef SAW_UNIX + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include "io.h" + +namespace saw { +namespace unix { +constexpr int MAX_EPOLL_EVENTS = 256; + +class unix_event_port; +class i_fd_owner { +protected: + unix_event_port &event_port_; + +private: + int file_descriptor_; + int fd_flags_; + uint32_t event_mask_; + +public: + i_fd_owner(unix_event_port &event_port, int file_descriptor, int fd_flags, + uint32_t event_mask); + + virtual ~i_fd_owner(); + + virtual void notify(uint32_t mask) = 0; + + int fd() const { return file_descriptor_; } +}; + +class unix_event_port final : public event_port { +private: + int epoll_fd_; + int signal_fd_; + + sigset_t signal_fd_set_; + + std::unordered_multimap>> + signal_conveyors_; + + int pipefds_[2]; + + std::vector to_unix_signal(Signal signal) const { + switch (signal) { + case Signal::User1: + return {SIGUSR1}; + case Signal::Terminate: + default: + return {SIGTERM, SIGQUIT, SIGINT}; + } + } + + Signal from_unix_signal(int signal) const { + switch (signal) { + case SIGUSR1: + return Signal::User1; + case SIGTERM: + case SIGINT: + case SIGQUIT: + default: + return Signal::Terminate; + } + } + + void notify_signal_listener(int sig) { + Signal signal = from_unix_signal(sig); + + auto equal_range = signal_conveyors_.equal_range(signal); + for (auto iter = equal_range.first; iter != equal_range.second; + ++iter) { + + if (iter->second) { + if (iter->second->space() > 0) { + iter->second->feed(); + } + } + } + } + + bool poll_impl(int time) { + epoll_event events[MAX_EPOLL_EVENTS]; + int nfds = 0; + do { + nfds = epoll_wait(epoll_fd_, events, MAX_EPOLL_EVENTS, time); + + if (nfds < 0) { + /// @todo error_handling + return false; + } + + for (int i = 0; i < nfds; ++i) { + if (events[i].data.u64 == 0) { + while (1) { + struct ::signalfd_siginfo siginfo; + ssize_t n = + ::read(signal_fd_, &siginfo, sizeof(siginfo)); + if (n < 0) { + break; + } + assert(n == sizeof(siginfo)); + + notify_signal_listener(siginfo.ssi_signo); + } + } else if (events[i].data.u64 == 1) { + uint8_t i; + if (pipefds_[0] < 0) { + continue; + } + while (1) { + ssize_t n = ::recv(pipefds_[0], &i, sizeof(i), 0); + if (n < 0) { + break; + } + } + } else { + i_fd_owner *owner = + reinterpret_cast(events[i].data.ptr); + if (owner) { + owner->notify(events[i].events); + } + } + } + } while (nfds == MAX_EPOLL_EVENTS); + + return true; + } + +public: + unix_event_port() : epoll_fd_{-1}, signal_fd_{-1} { + ::signal(SIGPIPE, SIG_IGN); + + epoll_fd_ = ::epoll_create1(EPOLL_CLOEXEC); + if (epoll_fd_ < 0) { + return; + } + + ::sigemptyset(&signal_fd_set_); + signal_fd_ = + ::signalfd(-1, &signal_fd_set_, SFD_NONBLOCK | SFD_CLOEXEC); + if (signal_fd_ < 0) { + return; + } + + struct epoll_event event; + memset(&event, 0, sizeof(event)); + event.events = EPOLLIN; + event.data.u64 = 0; + ::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, signal_fd_, &event); + + int rc = ::pipe2(pipefds_, O_NONBLOCK | O_CLOEXEC); + if (rc < 0) { + return; + } + memset(&event, 0, sizeof(event)); + event.events = EPOLLIN; + event.data.u64 = 1; + ::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, pipefds_[0], &event); + } + + ~unix_event_port() { + ::close(epoll_fd_); + ::close(signal_fd_); + ::close(pipefds_[0]); + ::close(pipefds_[1]); + } + + conveyor on_signal(Signal signal) override { + auto caf = new_conveyor_and_feeder(); + + signal_conveyors_.insert(std::make_pair(signal, std::move(caf.feeder))); + + std::vector sig = to_unix_signal(signal); + + for (auto iter = sig.begin(); iter != sig.end(); ++iter) { + ::sigaddset(&signal_fd_set_, *iter); + } + ::sigprocmask(SIG_BLOCK, &signal_fd_set_, nullptr); + ::signalfd(signal_fd_, &signal_fd_set_, SFD_NONBLOCK | SFD_CLOEXEC); + + auto node = conveyor::from_conveyor(std::move(caf.conveyor)); + return conveyor::to_conveyor(std::move(node)); + } + + void poll() override { poll_impl(0); } + + void wait() override { poll_impl(-1); } + + void wait(const std::chrono::steady_clock::duration &duration) override { + poll_impl( + std::chrono::duration_cast(duration) + .count()); + } + void + wait(const std::chrono::steady_clock::time_point &time_point) override { + auto now = std::chrono::steady_clock::now(); + if (time_point <= now) { + poll(); + } else { + poll_impl(std::chrono::duration_cast( + time_point - now) + .count()); + } + } + + void wake() override { + /// @todo pipe() in the beginning and write something minor into it like + /// uint8_t or sth the value itself doesn't matter + if (pipefds_[1] < 0) { + return; + } + uint8_t i = 0; + ::send(pipefds_[1], &i, sizeof(i), MSG_DONTWAIT); + } + + void subscribe(i_fd_owner &owner, int fd, uint32_t event_mask) { + if (epoll_fd_ < 0 || fd < 0) { + return; + } + ::epoll_event event; + memset(&event, 0, sizeof(event)); + event.events = event_mask | EPOLLET; + event.data.ptr = &owner; + + if (::epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &event) < 0) { + /// @todo error_handling + return; + } + } + + void unsubscribe(int fd) { + if (epoll_fd_ < 0 || fd < 0) { + return; + } + if (::epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, nullptr) < 0) { + /// @todo error_handling + return; + } + } +}; + +ssize_t unix_read(int fd, void *buffer, size_t length); +ssize_t unix_write(int fd, const void *buffer, size_t length); + +class unix_io_stream final : public io_stream, public i_fd_owner { +private: + own> read_ready_ = nullptr; + own> on_read_disconnect_ = nullptr; + own> write_ready_ = nullptr; + +public: + unix_io_stream(unix_event_port &event_port, int file_descriptor, + int fd_flags, uint32_t event_mask); + + error_or read(void *buffer, size_t length) override; + + conveyor read_ready() override; + + conveyor on_read_disconnected() override; + + error_or write(const void *buffer, size_t length) override; + + conveyor write_ready() override; + + /* + void read(void *buffer, size_t min_length, size_t max_length) override; + Conveyor readDone() override; + Conveyor readReady() override; + + Conveyor onReadDisconnected() override; + + void write(const void *buffer, size_t length) override; + Conveyor writeDone() override; + Conveyor writeReady() override; + */ + + void notify(uint32_t mask) override; +}; + +class unix_server final : public server, public i_fd_owner { +private: + own>> accept_feeder_ = nullptr; + +public: + unix_server(unix_event_port &event_port, int file_descriptor, int fd_flags); + + conveyor> accept() override; + + void notify(uint32_t mask) override; +}; + +class unix_datagram final : public datagram, public i_fd_owner { +private: + own> read_ready_ = nullptr; + own> write_ready_ = nullptr; + +public: + unix_datagram(unix_event_port &event_port, int file_descriptor, + int fd_flags); + + error_or read(void *buffer, size_t length) override; + conveyor read_ready() override; + + error_or write(const void *buffer, size_t length, + network_address &dest) override; + conveyor write_ready() override; + + void notify(uint32_t mask) override; +}; + +/** + * Helper class which provides potential addresses to NetworkAddress + */ +class socket_address { +private: + union { + struct sockaddr generic; + struct sockaddr_un unix; + struct sockaddr_in inet; + struct sockaddr_in6 inet6; + struct sockaddr_storage storage; + } address_; + + socklen_t address_length_; + bool wildcard_; + + socket_address() : wildcard_{false} {} + +public: + socket_address(const void *sockaddr, socklen_t len, bool wildcard) + : address_length_{len}, wildcard_{wildcard} { + assert(len <= sizeof(address_)); + memcpy(&address_.generic, sockaddr, len); + } + + int socket(int type) const { + type |= SOCK_NONBLOCK | SOCK_CLOEXEC; + + int result = ::socket(address_.generic.sa_family, type, 0); + return result; + } + + bool bind(int fd) const { + if (wildcard_) { + int value = 0; + ::setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &value, sizeof(value)); + } + int error = ::bind(fd, &address_.generic, address_length_); + return error < 0; + } + + struct ::sockaddr *get_raw() { + return &address_.generic; + } + + const struct ::sockaddr *get_raw() const { return &address_.generic; } + + socklen_t get_raw_length() const { return address_length_; } + + static std::vector resolve(std::string_view str, + uint16_t port_hint) { + std::vector results; + + struct ::addrinfo *head; + struct ::addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + + std::string port_string = std::to_string(port_hint); + bool wildcard = str == "*" || str == "::"; + std::string address_string{str}; + + int error = ::getaddrinfo(address_string.c_str(), port_string.c_str(), + &hints, &head); + + if (error) { + return {}; + } + + for (struct ::addrinfo *it = head; it != nullptr; it = it->ai_next) { + if (it->ai_addrlen > sizeof(socket_address::address_)) { + continue; + } + results.push_back({it->ai_addr, it->ai_addrlen, wildcard}); + } + ::freeaddrinfo(head); + return results; + } +}; + +class unix_network_address final : public os_network_address { +private: + const std::string path_; + uint16_t port_hint_; + std::vector addresses_; + +public: + unix_network_address(const std::string &path, uint16_t port_hint, + std::vector &&addr) + : path_{path}, port_hint_{port_hint}, addresses_{std::move(addr)} {} + + const std::string &address() const override; + + uint16_t port() const override; + + // Custom address info + socket_address &unix_address(size_t i = 0); + size_t unix_address_size() const; +}; + +class unix_network final : public network { +private: + unix_event_port &event_port_; + +public: + unix_network(unix_event_port &event_port); + + conveyor> + resolve_address(const std::string &address, + uint16_t port_hint = 0) override; + + own listen(network_address &addr) override; + + conveyor> connect(network_address &addr) override; + + own datagram(network_address &addr) override; +}; + +class unix_io_provider final : public io_provider { +private: + unix_event_port &event_port_; + class event_loop event_loop_; + + unix_network unix_network_; + +public: + unix_io_provider(unix_event_port &port_ref, own port); + + class network &get_network() override; + + own wrap_input_fd(int fd) override; + + class event_loop &event_loop(); +}; + +i_fd_owner::i_fd_owner(unix_event_port &event_port, int file_descriptor, + int fd_flags, uint32_t event_mask) + : event_port_{event_port}, file_descriptor_{file_descriptor}, + fd_flags_{fd_flags}, event_mask_{event_mask} { + event_port_.subscribe(*this, file_descriptor, event_mask); +} + +i_fd_owner::~i_fd_owner() { + if (file_descriptor_ >= 0) { + event_port_.unsubscribe(file_descriptor_); + ::close(file_descriptor_); + } +} + +ssize_t unix_read(int fd, void *buffer, size_t length) { + return ::recv(fd, buffer, length, 0); +} + +ssize_t unix_write(int fd, const void *buffer, size_t length) { + return ::send(fd, buffer, length, 0); +} + +unix_io_stream::unix_io_stream(unix_event_port &event_port, int file_descriptor, + int fd_flags, uint32_t event_mask) + : i_fd_owner{event_port, file_descriptor, fd_flags, + event_mask | EPOLLRDHUP} {} + +error_or unix_io_stream::read(void *buffer, size_t length) { + ssize_t read_bytes = unix_read(fd(), buffer, length); + if (read_bytes > 0) { + return static_cast(read_bytes); + } else if (read_bytes == 0) { + return make_error(); + } + + return make_error(); +} + +conveyor unix_io_stream::read_ready() { + auto caf = new_conveyor_and_feeder(); + read_ready_ = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +conveyor unix_io_stream::on_read_disconnected() { + auto caf = new_conveyor_and_feeder(); + on_read_disconnect_ = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +error_or unix_io_stream::write(const void *buffer, size_t length) { + ssize_t write_bytes = unix_write(fd(), buffer, length); + if (write_bytes > 0) { + return static_cast(write_bytes); + } + + int error = errno; + + if (error == EAGAIN || error == EWOULDBLOCK) { + return make_error(); + } + + return make_error(); +} + +conveyor unix_io_stream::write_ready() { + auto caf = new_conveyor_and_feeder(); + write_ready_ = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +void unix_io_stream::notify(uint32_t mask) { + if (mask & EPOLLOUT) { + if (write_ready_) { + write_ready_->feed(); + } + } + + if (mask & EPOLLIN) { + if (read_ready_) { + read_ready_->feed(); + } + } + + if (mask & EPOLLRDHUP) { + if (on_read_disconnect_) { + on_read_disconnect_->feed(); + } + } +} + +unix_server::unix_server(unix_event_port &event_port, int file_descriptor, + int fd_flags) + : i_fd_owner{event_port, file_descriptor, fd_flags, EPOLLIN} {} + +conveyor> unix_server::accept() { + auto caf = new_conveyor_and_feeder>(); + accept_feeder_ = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +void unix_server::notify(uint32_t mask) { + if (mask & EPOLLIN) { + if (accept_feeder_) { + struct ::sockaddr_storage address; + socklen_t address_length = sizeof(address); + + int accept_fd = + ::accept4(fd(), reinterpret_cast(&address), + &address_length, SOCK_NONBLOCK | SOCK_CLOEXEC); + if (accept_fd < 0) { + return; + } + auto fd_stream = heap(event_port_, accept_fd, 0, + EPOLLIN | EPOLLOUT); + accept_feeder_->feed(std::move(fd_stream)); + } + } +} + +unix_datagram::unix_datagram(unix_event_port &event_port, int file_descriptor, + int fd_flags) + : i_fd_owner{event_port, file_descriptor, fd_flags, EPOLLIN | EPOLLOUT} {} + +namespace { +ssize_t unix_read_msg(int fd, void *buffer, size_t length) { + struct ::sockaddr_storage their_addr; + socklen_t addr_len = sizeof(sockaddr_storage); + return ::recvfrom(fd, buffer, length, 0, + reinterpret_cast(&their_addr), + &addr_len); +} + +ssize_t unix_write_msg(int fd, const void *buffer, size_t length, + ::sockaddr *dest_addr, socklen_t dest_addr_len) { + + return ::sendto(fd, buffer, length, 0, dest_addr, dest_addr_len); +} +} // namespace + +error_or unix_datagram::read(void *buffer, size_t length) { + ssize_t read_bytes = unix_read_msg(fd(), buffer, length); + if (read_bytes > 0) { + return static_cast(read_bytes); + } + return make_error(); +} + +conveyor unix_datagram::read_ready() { + auto caf = new_conveyor_and_feeder(); + read_ready_ = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +error_or unix_datagram::write(const void *buffer, size_t length, + network_address &dest) { + unix_network_address &unix_dest = static_cast(dest); + socket_address &sock_addr = unix_dest.unix_address(); + socklen_t sock_addr_length = sock_addr.get_raw_length(); + ssize_t write_bytes = unix_write_msg(fd(), buffer, length, + sock_addr.get_raw(), sock_addr_length); + if (write_bytes > 0) { + return static_cast(write_bytes); + } + return make_error(); +} + +conveyor unix_datagram::write_ready() { + auto caf = new_conveyor_and_feeder(); + write_ready_ = std::move(caf.feeder); + return std::move(caf.conveyor); +} + +void unix_datagram::notify(uint32_t mask) { + if (mask & EPOLLOUT) { + if (write_ready_) { + write_ready_->feed(); + } + } + + if (mask & EPOLLIN) { + if (read_ready_) { + read_ready_->feed(); + } + } +} + +namespace { +bool begins_with(const std::string_view &viewed, + const std::string_view &begins) { + return viewed.size() >= begins.size() && + viewed.compare(0, begins.size(), begins) == 0; +} + +std::variant +translate_network_address_to_unix_network_address(network_address &addr) { + auto addr_variant = addr.representation(); + std::variant os_addr = + std::visit( + [](auto &arg) + -> std::variant { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + return static_cast(arg); + } + + auto sock_addrs = socket_address::resolve( + std::string_view{arg->address()}, arg->port()); + + return unix_network_address{arg->address(), arg->port(), + std::move(sock_addrs)}; + }, + addr_variant); + return os_addr; +} + +unix_network_address &translate_to_unix_address_ref( + std::variant &addr_variant) { + return std::visit( + [](auto &arg) -> unix_network_address & { + using T = std::decay_t; + + if constexpr (std::is_same_v) { + return arg; + } else if constexpr (std::is_same_v) { + return *arg; + } else { + static_assert(true, "Cases exhausted"); + } + }, + addr_variant); +} + +} // namespace + +own unix_network::listen(network_address &addr) { + auto unix_addr_storage = + translate_network_address_to_unix_network_address(addr); + unix_network_address &address = + translate_to_unix_address_ref(unix_addr_storage); + + assert(address.unix_address_size() > 0); + if (address.unix_address_size() == 0) { + return nullptr; + } + + int fd = address.unix_address(0).socket(SOCK_STREAM); + if (fd < 0) { + return nullptr; + } + + int val = 1; + int rc = ::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + if (rc < 0) { + ::close(fd); + return nullptr; + } + + bool failed = address.unix_address(0).bind(fd); + if (failed) { + ::close(fd); + return nullptr; + } + + ::listen(fd, SOMAXCONN); + + return heap(event_port_, fd, 0); +} + +conveyor> unix_network::connect(network_address &addr) { + auto unix_addr_storage = + translate_network_address_to_unix_network_address(addr); + unix_network_address &address = + translate_to_unix_address_ref(unix_addr_storage); + + assert(address.unix_address_size() > 0); + if (address.unix_address_size() == 0) { + return conveyor>{make_error()}; + } + + int fd = address.unix_address(0).socket(SOCK_STREAM); + if (fd < 0) { + return conveyor>{make_error()}; + } + + own io_str = + heap(event_port_, fd, 0, EPOLLIN | EPOLLOUT); + + bool success = false; + for (size_t i = 0; i < address.unix_address_size(); ++i) { + socket_address &addr_iter = address.unix_address(i); + int status = + ::connect(fd, addr_iter.get_raw(), addr_iter.get_raw_length()); + if (status < 0) { + int error = errno; + /* + * It's not connected yet... + * But edge triggered epolling means that it'll + * be ready when the signal is triggered + */ + + /// @todo Add limit node when implemented + if (error == EINPROGRESS) { + /* + Conveyor write_ready = io_stream->writeReady(); + return write_ready.then( + [ios{std::move(io_stream)}]() mutable { + ios->write_ready = nullptr; + return std::move(ios); + }); + */ + success = true; + break; + } else if (error != EINTR) { + /// @todo Push error message from + return conveyor>{make_error()}; + } + } else { + success = true; + break; + } + } + + if (!success) { + return conveyor>{make_error()}; + } + + return conveyor>{std::move(io_str)}; +} + +own unix_network::datagram(network_address &addr) { + auto unix_addr_storage = + translate_network_address_to_unix_network_address(addr); + unix_network_address &address = + translate_to_unix_address_ref(unix_addr_storage); + + SAW_ASSERT(address.unix_address_size() > 0) { return nullptr; } + + int fd = address.unix_address(0).socket(SOCK_DGRAM); + + int optval = 1; + int rc = + ::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); + if (rc < 0) { + ::close(fd); + return nullptr; + } + + bool failed = address.unix_address(0).bind(fd); + if (failed) { + ::close(fd); + return nullptr; + } + /// @todo + return heap(event_port_, fd, 0); +} + +const std::string &unix_network_address::address() const { return path_; } + +uint16_t unix_network_address::port() const { return port_hint_; } + +socket_address &unix_network_address::unix_address(size_t i) { + assert(i < addresses_.size()); + /// @todo change from list to vector? + return addresses_.at(i); +} + +size_t unix_network_address::unix_address_size() const { + return addresses_.size(); +} + +unix_network::unix_network(unix_event_port &event) : event_port_{event} {} + +conveyor> +unix_network::resolve_address(const std::string &path, uint16_t port_hint) { + std::string_view addr_view{path}; + { + std::string_view str_begins_with = "unix:"; + if (begins_with(addr_view, str_begins_with)) { + addr_view.remove_prefix(str_begins_with.size()); + } + } + + std::vector addresses = + socket_address::resolve(addr_view, port_hint); + + return conveyor>{ + heap(path, port_hint, std::move(addresses))}; +} + +unix_io_provider::unix_io_provider(unix_event_port &port_ref, + own port) + : event_port_{port_ref}, event_loop_{std::move(port)}, unix_network_{ + port_ref} {} + +own unix_io_provider::wrap_input_fd(int fd) { + return heap(event_port_, fd, 0, EPOLLIN); +} + +class network &unix_io_provider::get_network() { + return static_cast(unix_network_); +} + +class event_loop &unix_io_provider::event_loop() { + return event_loop_; +} + +} // namespace unix + +error_or setup_async_io() { + using namespace unix; + try { + own prt = heap(); + unix_event_port &prt_ref = *prt; + + own io_provider = + heap(prt_ref, std::move(prt)); + + event_loop &loop_ref = io_provider->event_loop(); + + return {{std::move(io_provider), loop_ref, prt_ref}}; + } catch (std::bad_alloc &) { + return make_error(); + } +} +} // namespace saw +#endif diff --git a/modules/io_codec/.nix/derivation.nix b/modules/io_codec/.nix/derivation.nix new file mode 100644 index 0000000..7cd55a8 --- /dev/null +++ b/modules/io_codec/.nix/derivation.nix @@ -0,0 +1,31 @@ +{ lib +, stdenv +, scons +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-io_codec"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.io + forstio.codec + ]; + + outputs = ["out" "dev"]; +} diff --git a/modules/io_codec/SConscript b/modules/io_codec/SConscript new file mode 100644 index 0000000..0afd6d6 --- /dev/null +++ b/modules/io_codec/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +io_env = env.Clone(); + +io_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +io_env.headers = sorted(glob.glob(dir_path + "/*.h")) + +env.sources += io_env.sources; +env.headers += io_env.headers; + +## Shared lib +objects_shared = [] +io_env.add_source_files(objects_shared, io_env.sources, shared=True); +io_env.library_shared = io_env.SharedLibrary('#build/forstio-io_codec', [objects_shared]); + +## Static lib +objects_static = [] +io_env.add_source_files(objects_static, io_env.sources, shared=False); +io_env.library_static = io_env.StaticLibrary('#build/forstio-io_codec', [objects_static]); + +# Set Alias +env.Alias('library_io_codec', [io_env.library_shared, io_env.library_static]); + +env.targets += ['library_io_codec']; + +# Install +env.Install('$prefix/lib/', [io_env.library_shared, io_env.library_static]); +env.Install('$prefix/include/forstio/io_codec/', [io_env.headers]); diff --git a/modules/io_codec/SConstruct b/modules/io_codec/SConstruct new file mode 100644 index 0000000..4e6e150 --- /dev/null +++ b/modules/io_codec/SConstruct @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=['forstio-io']) +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/io_codec/io_peer.h b/modules/io_codec/io_peer.h new file mode 100644 index 0000000..b9a4b34 --- /dev/null +++ b/modules/io_codec/io_peer.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include +#include + +namespace saw { + +template , + typename OutContainer = message_container, + typename BufferT = ring_buffer> +class streaming_io_peer { +public: + /** + * + */ + streaming_io_peer( + own>> feed, + own stream, Codec codec, BufferT in, BufferT out); + /** + * + */ + streaming_io_peer( + own>> feed, + own stream); + + /** + * Deleted copy and move constructors + */ + SAW_FORBID_COPY(streaming_io_peer); + SAW_FORBID_MOVE(streaming_io_peer); + + /** + * Send a message to the remote peer + */ + error send(heap_message_root builder); + + /** + * A phantom conveyor feeder. Meant for interfacing with other components + */ + conveyor_feeder> &feeder(); + + conveyor on_read_disconnected(); + +private: + /// @unimplemented + class peer_conveyor_feeder final + : public conveyor_feeder> { + public: + peer_conveyor_feeder( + streaming_io_peer &peer_) + : peer_{peer_} {} + + void feed(heap_message_root &&data) override { + (void)data; + } + + void fail(error &&error) override { (void)error; } + + size_t space() const override { return 0; } + + size_t queued() const override { return 0; } + + private: + streaming_io_peer &peer_; + }; + +private: + own>> + incoming_feeder_ = nullptr; + + own io_stream_; + + Codec codec_; + + BufferT in_buffer_; + BufferT out_buffer_; + + conveyor_sink sink_read_; + conveyor_sink sink_write_; + + peer_conveyor_feeder conveyor_feeder_; +}; + +/** + * Setup new streaming io peer with the provided network protocols. + * This is a convenience wrapper intended for a faster setup of this class + */ +template , + typename OutContainer = message_container, + typename BufferT = ring_buffer> +std::pair>, + conveyor>> +new_streaming_io_peer(own stream); + +} // namespace saw + +#include "io_peer.tmpl.h" diff --git a/modules/io_codec/io_peer.tmpl.h b/modules/io_codec/io_peer.tmpl.h new file mode 100644 index 0000000..880a58a --- /dev/null +++ b/modules/io_codec/io_peer.tmpl.h @@ -0,0 +1,117 @@ +namespace saw { + +template +streaming_io_peer:: + streaming_io_peer( + own>> feed, + own str) + : streaming_io_peer{std::move(feed), std::move(str), {}, {}, {}} {} + +template +streaming_io_peer:: + streaming_io_peer( + own>> feed, + own stream, Codec codec, BufferT in, BufferT out) + : incoming_feeder_{std::move(feed)}, + io_stream_{std::move(stream)}, codec_{std::move(codec)}, + in_buffer_{std::move(in)}, out_buffer_{std::move(out)}, + sink_read_{ + io_stream_->read_done() + .then([this](size_t bytes) -> error_or { + in_buffer_.write_advance(bytes); + + if (in_buffer_.write_segment_length() == 0) { + return critical_error("Message too long"); + } + + io_stream_->read(&in_buffer_.write(), 1, + in_buffer_.write_segment_length()); + + while (true) { + auto root = heap_message_root(); + auto builder = root.build(); + + error err = codec_.template decode( + builder, in_buffer_); + if (err.is_critical()) { + return err; + } + + if (!err.failed()) { + incoming_feeder_->feed(std::move(root)); + } else { + break; + } + } + + return void_t{}; + }) + .sink([this](error err) { + incoming_feeder_->fail(err.copy_error()); + + return err; + })}, + sink_write_{io_stream_->write_done() + .then([this](size_t bytes) -> error_or { + out_buffer_.read_advance(bytes); + if (out_buffer_.readCompositeLength() > 0) { + io_stream_->write( + &out_buffer_.read(), + out_buffer_.read_segment_length()); + } + + return void_t{}; + }) + .sink()} { + io_stream_->read(&in_buffer_.write(), 1, in_buffer_.write_segment_length()); +} + +template +error streaming_io_peer::send(heap_message_root + msg) { + bool restart_write = out_buffer_.read_segment_length() == 0; + + error err = + codec_.template encode(msg.read(), out_buffer_); + if (err.failed()) { + return err; + } + + if (restart_write) { + io_stream_->write(&out_buffer_.read(), + out_buffer_.read_segment_length()); + } + + return no_error(); +} + +template +conveyor +streaming_io_peer::on_read_disconnected() { + return io_stream_->on_read_disconnected(); +} + +template +std::pair>, + conveyor>> +newstreaming_io_peer(own stream) { + auto caf = + new_conveyor_and_feeder>(); + + return {heap>( + std::move(caf.feeder), std::move(stream)), + std::move(caf.conveyor)}; +} + +} // namespace saw diff --git a/modules/io_codec/rpc.h b/modules/io_codec/rpc.h new file mode 100644 index 0000000..020bf96 --- /dev/null +++ b/modules/io_codec/rpc.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +namespace saw { +namespace rmt { +struct Network {}; +} + +template<> +class remote { +private: + std::string addr_str_; +public: + remote(std::string addr_str); + + template + conveyor> create_client(); +}; + +template