From fe53a6cc42b645cab62515e257f52426ae568753 Mon Sep 17 00:00:00 2001 From: "Claudius \"keldu\" Holeksa" Date: Mon, 7 Apr 2025 14:20:42 +0200 Subject: Copied netcdf for vtk preparation --- modules/codec-vtk/.nix/derivation.nix | 37 ++++ modules/codec-vtk/SConstruct | 73 +++++++ modules/codec-vtk/c++/SConscript | 38 ++++ modules/codec-vtk/c++/netcdf.hpp | 148 +++++++++++++ modules/codec-vtk/c++/netcdf.tmpl.hpp | 348 +++++++++++++++++++++++++++++++ modules/codec-vtk/tests/SConscript | 31 +++ modules/codec-vtk/tests/codec-netcdf.cpp | 104 +++++++++ 7 files changed, 779 insertions(+) create mode 100644 modules/codec-vtk/.nix/derivation.nix create mode 100644 modules/codec-vtk/SConstruct create mode 100644 modules/codec-vtk/c++/SConscript create mode 100644 modules/codec-vtk/c++/netcdf.hpp create mode 100644 modules/codec-vtk/c++/netcdf.tmpl.hpp create mode 100644 modules/codec-vtk/tests/SConscript create mode 100644 modules/codec-vtk/tests/codec-netcdf.cpp diff --git a/modules/codec-vtk/.nix/derivation.nix b/modules/codec-vtk/.nix/derivation.nix new file mode 100644 index 0000000..00f610d --- /dev/null +++ b/modules/codec-vtk/.nix/derivation.nix @@ -0,0 +1,37 @@ +{ lib +, stdenv +, scons +, clang +, clang-tools +, version +, forstio +}: + +let + +in stdenv.mkDerivation { + pname = "forstio-codec-vtk"; + inherit version; + src = ./..; + + enableParallelBuilding = true; + + nativeBuildInputs = [ + scons + clang-tools + ]; + + buildInputs = [ + forstio.core + forstio.async + forstio.codec + ]; + + doCheck = true; + checkPhase = '' + scons test + ./bin/tests + ''; + + outputs = ["out" "dev"]; +} diff --git a/modules/codec-vtk/SConstruct b/modules/codec-vtk/SConstruct new file mode 100644 index 0000000..adabe4b --- /dev/null +++ b/modules/codec-vtk/SConstruct @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +import sys +import os +import os.path +import glob +import re + + +if sys.version_info < (3,): + def isbasestring(s): + return isinstance(s,basestring) +else: + def isbasestring(s): + return isinstance(s, (str,bytes)) + +def add_kel_source_files(self, sources, filetype, lib_env=None, shared=False, target_post=""): + + if isbasestring(filetype): + dir_path = self.Dir('.').abspath + filetype = sorted(glob.glob(dir_path+"/"+filetype)) + + for path in filetype: + target_name = re.sub( r'(.*?)(\.cpp|\.c\+\+)', r'\1' + target_post, path ) + if shared: + target_name+='.os' + sources.append( self.SharedObject( target=target_name, source=path ) ) + else: + target_name+='.o' + sources.append( self.StaticObject( target=target_name, source=path ) ) + pass + +def isAbsolutePath(key, dirname, env): + assert os.path.isabs(dirname), "%r must have absolute path syntax" % (key,) + +env_vars = Variables( + args=ARGUMENTS +) + +env_vars.Add('prefix', + help='Installation target location of build results and headers', + default='/usr/local/', + validator=isAbsolutePath +) + +env=Environment(ENV=os.environ, variables=env_vars, CPPPATH=[], + CXX=['c++'], + CPPDEFINES=['SAW_UNIX'], + CXXFLAGS=['-std=c++20','-g','-Wall','-Wextra'], + LIBS=[ + 'forstio-core', + 'forstio-codec', + 'netcdf' + ] +); +env.__class__.add_source_files = add_kel_source_files +env.Tool('compilation_db'); +env.cdb = env.CompilationDatabase('compile_commands.json'); + +env.objects = []; +env.sources = []; +env.headers = []; +env.targets = []; + +Export('env') +SConscript('c++/SConscript') +SConscript('tests/SConscript') + +env.Alias('cdb', env.cdb); +env.Alias('all', [env.targets]); +env.Default('all'); + +env.Alias('install', '$prefix') diff --git a/modules/codec-vtk/c++/SConscript b/modules/codec-vtk/c++/SConscript new file mode 100644 index 0000000..6d8866b --- /dev/null +++ b/modules/codec-vtk/c++/SConscript @@ -0,0 +1,38 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +codec_netcdf_env = env.Clone(); + +codec_netcdf_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +codec_netcdf_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += codec_netcdf_env.sources; +env.headers += codec_netcdf_env.headers; + +## Shared lib +objects_shared = [] +codec_netcdf_env.add_source_files(objects_shared, codec_netcdf_env.sources, shared=True); +env.library_shared = codec_netcdf_env.SharedLibrary('#build/forstio-codec-netcdf', [objects_shared]); + +## Static lib +objects_static = [] +codec_netcdf_env.add_source_files(objects_static, codec_netcdf_env.sources, shared=False); +env.library_static = codec_netcdf_env.StaticLibrary('#build/forstio-codec-netcdf', [objects_static]); + +# Set Alias +env.Alias('library_codec_netcdf', [env.library_shared, env.library_static]); + +env.targets += ['library_codec_netcdf']; + +# Install +env.Install('$prefix/lib/', [env.library_shared, env.library_static]); +env.Install('$prefix/include/forstio/codec/netcdf/', [codec_netcdf_env.headers]); diff --git a/modules/codec-vtk/c++/netcdf.hpp b/modules/codec-vtk/c++/netcdf.hpp new file mode 100644 index 0000000..2c0a786 --- /dev/null +++ b/modules/codec-vtk/c++/netcdf.hpp @@ -0,0 +1,148 @@ +#pragma once + +#include +#include +#include + +namespace saw { +namespace encode { +/** + * Template type hint + */ +struct VtkLegacy {}; +} + +/** + * Class representing the files system netcdf file + */ +template +class data { +private: + std::vector buff_; +public: + data() = default; + + data(std::vector buff): + buff_{std::move(buff)} + {} + + template + data(const std::array& arr): + buff_{arr.begin(), arr.end()} + {} + + std::vector& get_data() { + return buff_; + } + + const std::vector& get_data() const { + return buff_; + } +}; + +template +class codec{ + static_assert(always_false, "NetCDF only supports Structs as a root object"); +}; + +} +#include "vtk.tmpl.hpp" +namespace saw { + +template +class codec...>, encode::VtkLegacy> { +private: + using Schema = schema::Struct...>; +public: + SAW_FORBID_COPY(codec); + SAW_DEFAULT_MOVE(codec); + + /** + * Default constructor + */ + codec() = default; + + /** + * Encoder function + */ + template + error_or encode(const data& from, data& to) { + int rc{}; + int ncid{}; + + rc = nc_create_mem("forstio_internal_write_memory", 0,/*NC_NETCDF4, */0, &ncid); + if(rc != NC_NOERR){ + return make_error(); + } + + { + auto eov = impl::netcdf_encode::encode_meta(from, ncid); + if(eov.is_error()){ + nc_close(ncid); + return eov; + } + } + + rc = nc_enddef(ncid); + if(rc != NC_NOERR){ + nc_close(ncid); + return make_error(); + } + + { + auto eov = impl::netcdf_encode::encode(from, ncid); + if(eov.is_error()){ + nc_close(ncid); + return eov; + } + } + + NC_memio nc_memio; + rc = nc_close_memio(ncid, &nc_memio); + if(rc != NC_NOERR){ + return make_error(); + } + + if(!nc_memio.memory || nc_memio.size == 0){ + return make_error(); + } + + try { + to.get_data().resize(nc_memio.size); + }catch(const std::exception& e){ + (void) e; + return make_error(); + } + + for(std::size_t i = 0; i < nc_memio.size; ++i){ + to.get_data().at(i) = static_cast(nc_memio.memory)[i]; + } + + free(nc_memio.memory); + nc_memio.memory = nullptr; + + return void_t{}; + } + + /** + * Decoder function + */ + template + error_or decode(data& from_decode, data& to_decode) { + int ncid{}; + int rc{}; + + rc = nc_open_mem("forstio_internal_read_memory", NC_NOWRITE, from_decode.get_data().size(), &from_decode.get_data()[0], &ncid); + if(rc != NC_NOERR){ + // Don't know how to get the error, so fail critically. + return make_error(); + } + + auto eov = impl::netcdf_decode::decode(to_decode, ncid); + + nc_close(ncid); + + return eov; + } +}; +} diff --git a/modules/codec-vtk/c++/netcdf.tmpl.hpp b/modules/codec-vtk/c++/netcdf.tmpl.hpp new file mode 100644 index 0000000..bf257e4 --- /dev/null +++ b/modules/codec-vtk/c++/netcdf.tmpl.hpp @@ -0,0 +1,348 @@ +#pragma once + +namespace saw { +namespace impl { +template +struct netcdf_primitive_id { + static_assert(always_false, "Not a primitive id"); +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_INT; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_UINT; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_INT64; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_UINT64; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_FLOAT; +}; + +template<> +struct netcdf_primitive_id { + static constexpr nc_type value = NC_DOUBLE; +}; + +template +struct netcdf_is_group { + static constexpr bool value = false; +}; + +template +struct netcdf_is_group...>> { + static constexpr bool value = true; +}; + +template +struct netcdf_encode; + +template +struct netcdf_encode, ToEncode> { + using Schema = schema::Primitive; + + static error_or encode(const data& from, int ncid, int ncvarid){ + int rc{}; + + typename native_data_type::type val = from.get(); + + rc = nc_put_var(ncid, ncvarid, &val); + if(rc != NC_NOERR) { + return make_error(); + } + + return void_t{}; + } + + static error_or encode_meta(const data& from, int ncid, int ncvarid){ + (void) from; + (void) ncid; + (void) ncvarid; + return void_t{}; + } +}; + +template +struct netcdf_encode, ToEncode> { + using Schema = schema::Array; + + template + static error_or encode_level(data& from, int ncid, int ncvarid){ + + return make_error(); + } + + static error_or encode(data& from, int ncid, int ncvarid){ + + return make_error(); + } + + static error_or encode_meta(const data& from, int ncid, int ncvarid){ + + return make_error(); + } +}; + +template +struct netcdf_encode...>, ToEncode> { + using Schema = schema::Struct...>; + + template + static error_or encode_member(const data& from, int ncid){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + /** + * We have to ask for the internal id of the netcdf structure + */ + if constexpr (netcdf_is_group::value){ + return make_error(); + }else{ + int nc_varid{}; + int rc {}; + rc = nc_inq_varid(ncid, Literal.data.data(), &nc_varid); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_encode::encode(from.template get(), ncid, nc_varid); + if(eov.is_error()){ + return eov; + } + } + + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = encode_member(from, ncid); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } + + static error_or encode(const data& from, int ncid){ + if constexpr (sizeof...(T) > 0){ + auto eov = encode_member<0>(from, ncid); + return eov; + } + + return void_t{}; + } + + template + static error_or encode_meta_member(const data& from, int ncid){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + /** + * We have to ask for the internal id of the netcdf structure + */ + if constexpr (netcdf_is_group::value){ + return make_error(); + }else{ + int nc_varid{}; + int rc {}; + rc = nc_def_var(ncid, Literal.data.data(), netcdf_primitive_id::value, 0, nullptr, &nc_varid); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_encode::encode_meta(from.template get(), ncid, nc_varid); + if(eov.is_error()){ + return eov; + } + } + + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = encode_meta_member(from, ncid); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } + + static error_or encode_meta(const data& from, int ncid){ + if constexpr (sizeof...(T) > 0){ + auto eov = encode_meta_member<0>(from, ncid); + return eov; + } + + return void_t{}; + } +}; + +template +struct netcdf_decode; + +template +struct netcdf_decode, ToDecode> { + using Schema = schema::Primitive; + + static error_or decode(data& to, int from, int nc_varid){ + int rc{}; + + nc_type nc_vartype{}; + int nc_dimnum{}; + std::array nc_dimids; + + rc = nc_inq_var(from, nc_varid, nullptr, &nc_vartype, &nc_dimnum, &nc_dimids[0], nullptr); + if(rc != NC_NOERR){ + return make_error(); + } + if(nc_vartype != netcdf_primitive_id>::value){ + return make_error(); + } + if(nc_dimnum != 0){ + return make_error(); + } + + typename native_data_type>::type val{}; + rc = nc_get_var(from, nc_varid, &val); + if(rc != NC_NOERR){ + return make_error(); + } + + to.set(val); + + return void_t {}; + } +}; + +template +struct netcdf_decode, ToDecode> { + using Schema = schema::Array; + + template + static error_or decode_level(data& to, int from, int nc_varid, std::array& index, const std::array& count){ + if constexpr ( Level == Dim ){ + int rc{}; + typename native_data_type::type val; + rc = nc_get_vara(from, nc_varid, index.data(), count.data(), &val); + if(rc != NC_NOERR){ + return make_error(); + } + to.at(index).set(val); + }else{ + const std::size_t dim_size = to.get_dim_size(Level); + for(index[Level] = 0; index[Level] < dim_size; ++index[Level]){ + auto eov = decode_level(to, from, nc_varid, index, count); + if(eov.is_error()){ + return eov; + } + } + } + + return void_t{}; + } + + static error_or decode(data& to, int from, int nc_varid){ + int rc{}; + + nc_type nc_vartype{}; + + int nc_dimnum{}; + std::array nc_dimids; + + rc = nc_inq_var(from, nc_varid, nullptr, &nc_vartype, &nc_dimnum, &nc_dimids[0], nullptr); + if(rc != NC_NOERR){ + return make_error(); + } + if(nc_vartype != netcdf_primitive_id::value){ + return make_error(); + } + if(nc_dimnum != Dim){ + return make_error(); + } + + std::array dims; + std::array count; + for(std::size_t i = 0; i < Dim; ++i){ + rc = nc_inq_dim(from, nc_dimids[i], nullptr, &dims[i]); + if(rc != NC_NOERR){ + return make_error(); + } + count[i] = 1; + } + + to = {dims}; + std::array index; + + return decode_level<0>(to, from, nc_varid, index, count); + } +}; + +template +struct netcdf_decode...>, ToDecode> { + using Schema = schema::Struct...>; + + template + static error_or decode_member(data& to, int from){ + using Type = typename parameter_pack_type::type; + constexpr string_literal Literal = parameter_key_pack_type::literal; + { + /** + * We have to ask for the internal id of the netcdf structure + */ + if constexpr (netcdf_is_group::value){ + int nc_group_id{}; + int rc {}; + rc = nc_inq_ncid(from, Literal.data.data(), &nc_group_id); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_decode::decode(to.template get(), nc_group_id); + if(eov.is_error()){ + return eov; + } + }else{ + int nc_varid{}; + int rc {}; + rc = nc_inq_varid(from, Literal.data.data(), &nc_varid); + if(rc != NC_NOERR) { + return make_error(); + } + auto eov = netcdf_decode::decode(to.template get(), from, nc_varid); + if(eov.is_error()){ + return eov; + } + } + + } + if constexpr ((i+1) < sizeof...(T)){ + auto eov = decode_member(to, from); + if(eov.is_error()){ + return eov; + } + } + + return void_t{}; + } + + + static error_or decode(data& to, int from){ + if constexpr (sizeof...(T) > 0){ + auto eov = decode_member<0>(to, from); + return eov; + } + + return void_t{}; + } +}; +} +} diff --git a/modules/codec-vtk/tests/SConscript b/modules/codec-vtk/tests/SConscript new file mode 100644 index 0000000..f8ffc92 --- /dev/null +++ b/modules/codec-vtk/tests/SConscript @@ -0,0 +1,31 @@ +#!/bin/false + +import os +import os.path +import glob + + +Import('env') + +dir_path = Dir('.').abspath + +# Environment for base library +test_cases_env = env.Clone(); + +test_cases_env.Append(LIBS=['forstio-test']); + +test_cases_env.sources = sorted(glob.glob(dir_path + "/*.cpp")) +test_cases_env.headers = sorted(glob.glob(dir_path + "/*.hpp")) + +env.sources += test_cases_env.sources; +env.headers += test_cases_env.headers; + +objects_static = [] +test_cases_env.add_source_files(objects_static, test_cases_env.sources, shared=False); +test_cases_env.program = test_cases_env.Program('#bin/tests', [objects_static, env.library_static]); + +# Set Alias +env.Alias('test', test_cases_env.program); +env.Alias('check', test_cases_env.program); + +env.targets += ['test','check']; diff --git a/modules/codec-vtk/tests/codec-netcdf.cpp b/modules/codec-vtk/tests/codec-netcdf.cpp new file mode 100644 index 0000000..540f60e --- /dev/null +++ b/modules/codec-vtk/tests/codec-netcdf.cpp @@ -0,0 +1,104 @@ +#include +#include "../c++/netcdf.hpp" + +namespace { +namespace schema { +using namespace saw::schema; +using TestStruct = Struct< + Member, + Member +>; + +using TestArrayStruct = Struct< + Member, "data"> +>; +} + +std::array tests_data_primitive_nc = { + 0x43, 0x44, 0x46, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x05, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x05, 0x40, 0x40, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 +}; + +std::array tests_data_array_nc = { + 0x43, 0x44, 0x46, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x78, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x79, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e +}; + +SAW_TEST("NetCDF Struct Primitive write"){ + using namespace saw; + + data native; + + native.template get<"data">().set(5); + native.template get<"other">().set(32.0); + + codec codec; + + data netcdf; + + auto eov = codec.encode(native, netcdf); + SAW_EXPECT(eov.is_value(), "Encoding failed"); + auto& nc_data = netcdf.get_data(); + SAW_EXPECT(nc_data.size() == tests_data_primitive_nc.size(), "Incorrect size"); + + for(std::size_t i = 0; i < nc_data.size(); ++i){ + SAW_EXPECT(nc_data.at(i) == tests_data_primitive_nc.at(i), "Incorrect Value"); + } +} + +SAW_TEST("NetCDF Struct Primitive read"){ + using namespace saw; + + data netcdf{tests_data_primitive_nc}; + data native; + + codec codec; + + auto eov = codec.decode(netcdf, native); + SAW_EXPECT(eov.is_value(), "Decoding failed"); + SAW_EXPECT(native.template get<"data">().get() == 5, "Int Value incorrect"); + SAW_EXPECT(native.template get<"other">().get() == 32.0, "Double Value incorrect"); +} + +SAW_TEST("NetCDF Struct Array read"){ + using namespace saw; + + data netcdf{tests_data_array_nc}; + + data native; + + codec codec; + + auto eov = codec.decode(netcdf, native); + SAW_EXPECT(eov.is_value(), "Decoding failed"); + auto& arr = native.template get<"data">(); + SAW_EXPECT(arr.get_dim_size(0) == 5, "Incorrect dimension 0"); + SAW_EXPECT(arr.get_dim_size(1) == 3, "Incorrect dimension 1"); + + for(std::size_t i = 0; i < 5; ++i){ + for(std::size_t j = 0; j < 3; ++j){ + int64_t exp_val = i * 3 + j; + SAW_EXPECT(arr.at({i,j}).get() == exp_val, "Incorrect value"); + } + } +} +} -- cgit v1.2.3