diff --git a/mapping_plugin/CMakeLists.txt b/mapping_plugin/CMakeLists.txt index 43390c0..4d38c34 100644 --- a/mapping_plugin/CMakeLists.txt +++ b/mapping_plugin/CMakeLists.txt @@ -8,6 +8,45 @@ find_package( UDA 2.7.0 REQUIRED ) find_package( LibTokaMap REQUIRED ) +# Optional embedded-Python data-source support. When ON, the plugin can serve +# data sources implemented in Python (e.g. tokamap_compass CDBDataSource) by +# embedding CPython in uda_server. The interpreter starts lazily — only when +# the libtokamap config declares a [python_data_sources] table — so a plugin +# built with this ON still runs Python-free until such a source is configured. +# When OFF the plugin has no Python dependency at all. +option( MAPPING_PLUGIN_PYTHON "Embed CPython to serve Python data sources" OFF ) + +set( MAPPING_PLUGIN_PYTHON_SOURCES "" ) +set( MAPPING_PLUGIN_PYTHON_INCLUDES "" ) +set( MAPPING_PLUGIN_PYTHON_DEFINITIONS "" ) +set( MAPPING_PLUGIN_PYTHON_LIBS "" ) +if( MAPPING_PLUGIN_PYTHON ) + find_package( Python3 REQUIRED COMPONENTS Interpreter Development NumPy ) + # Only a single dlopen-able libpython path is wanted here. Python3_LIBRARIES + # can be a multi-element list (and carries optimized;/debug; keywords on MSVC), + # which would split -DMAPPING_PLUGIN_DEFAULT_LIBPYTHON="..." across two + # compiler arguments and fail the compile on the unterminated quote. + set( MAPPING_PLUGIN_LIBPYTHON "" ) + foreach( LIB IN LISTS Python3_LIBRARIES ) + if( LIB MATCHES "python" AND EXISTS "${LIB}" ) + set( MAPPING_PLUGIN_LIBPYTHON "${LIB}" ) + break() + endif() + endforeach() + if( NOT MAPPING_PLUGIN_LIBPYTHON ) + message( WARNING + "Could not pick a libpython path out of '${Python3_LIBRARIES}'; the plugin will need " + "MAPPING_PLUGIN_LIBPYTHON set in the environment to start the embedded interpreter." ) + endif() + set( MAPPING_PLUGIN_PYTHON_DEFINITIONS -DMAPPING_PLUGIN_PYTHON + -DMAPPING_PLUGIN_DEFAULT_LIBPYTHON="${MAPPING_PLUGIN_LIBPYTHON}" ) + set( MAPPING_PLUGIN_PYTHON_INCLUDES ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS} ) + set( MAPPING_PLUGIN_PYTHON_LIBS Python3::Python ${CMAKE_DL_LIBS} ) +endif() +# Either way python_data_source.cpp is compiled: with MAPPING_PLUGIN_PYTHON it +# carries the real implementation, otherwise a stub that only fails loudly if +# the config actually requests Python sources. + include( plugins ) uda_plugin( NAME IMAS_MAP @@ -15,16 +54,20 @@ uda_plugin( DESCRIPTION "MAP IMAS element name to TOKAMAK data using JSON mapping files" EXAMPLE "IMAS_MAP::read()" LIBNAME mapping_plugin - SOURCES mapping_plugin.cpp src/uda_plugin_helpers.cpp + SOURCES mapping_plugin.cpp src/uda_plugin_helpers.cpp src/python_data_source.cpp CONFIG_FILE config/json_mapping.cfg config/libtokamap_config.toml EXTRA_INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/src ${CMAKE_CURRENT_LIST_DIR}/ext_include ${UDA_CLIENT_INCLUDE_DIRS} + ${MAPPING_PLUGIN_PYTHON_INCLUDES} + EXTRA_DEFINITIONS + ${MAPPING_PLUGIN_PYTHON_DEFINITIONS} EXTRA_LINK_DIRS ${UDA_CLIENT_LIBRARY_DIRS} EXTRA_LINK_LIBS LibTokaMap::libtokamap + ${MAPPING_PLUGIN_PYTHON_LIBS} ) # Generate scripts diff --git a/mapping_plugin/mapping_plugin.cpp b/mapping_plugin/mapping_plugin.cpp index baa92a1..789e2ce 100644 --- a/mapping_plugin/mapping_plugin.cpp +++ b/mapping_plugin/mapping_plugin.cpp @@ -31,6 +31,7 @@ #include #include "uda_plugin_helpers.hpp" +#include "python_data_source.hpp" namespace { @@ -88,6 +89,7 @@ class MappingPlugin static int max_interface_version(IDAM_PLUGIN_INTERFACE* plugin_interface); bool m_init = false; + bool m_python_init = false; std::string m_request_function; libtokamap::MappingHandler m_mapping_handler; }; @@ -113,6 +115,20 @@ int MappingPlugin::init(IDAM_PLUGIN_INTERFACE* plugin_interface) const char* config_path = getenv("UDA_MAPPING_CONFIG_PATH"); if (config_path != nullptr) { m_mapping_handler.init(std::filesystem::path{config_path}); + // Register any Python data sources / custom functions declared in the + // MAPPING_PLUGIN_PYTHON_CONFIG file. Starts the embedded interpreter + // only when such a config exists; no-op otherwise (and a compile-time + // no-op when built without MAPPING_PLUGIN_PYTHON). + // + // Done at most once per process. reset() clears m_init, but the mapping + // handler keeps its registered data sources (just as it keeps its + // experiment register, which is why its own init() is a no-op on the + // second call), so re-running this would re-import the Python modules + // and reconnect the sources for nothing. + if (!m_python_init) { + mapping_plugin::init_python_data_sources_if_configured(m_mapping_handler); + m_python_init = true; + } } else { throw std::runtime_error{"UDA_MAPPING_CONFIG_PATH not specified"}; } diff --git a/mapping_plugin/src/python_data_source.cpp b/mapping_plugin/src/python_data_source.cpp new file mode 100644 index 0000000..6a57e8c --- /dev/null +++ b/mapping_plugin/src/python_data_source.cpp @@ -0,0 +1,870 @@ +// Lazy embedded-Python data-source support for the mapping plugin. +// +// Built only when MAPPING_PLUGIN_PYTHON is defined (requires Python3 + NumPy +// headers). See python_data_source.hpp for the rationale and lifetime model. + +#include "python_data_source.hpp" + +#ifdef MAPPING_PLUGIN_PYTHON + +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION + +#include + +// Pull libpython into the global symbol namespace BEFORE Py_Initialize: +// UDA dlopens this plugin RTLD_LOCAL, so without this NumPy's C extensions +// cannot resolve Python symbols at import time. +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace mapping_plugin +{ +namespace +{ + +// --------------------------------------------------------------------------- +// Python error reporting +// --------------------------------------------------------------------------- + +std::string fetch_python_error() +{ + if (PyErr_Occurred() == nullptr) { + return {}; + } + PyObject *ptype = nullptr, *pvalue = nullptr, *ptraceback = nullptr; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + PyErr_NormalizeException(&ptype, &pvalue, &ptraceback); + std::string message{"unknown Python error"}; + if (pvalue != nullptr) { + PyObject* str = PyObject_Str(pvalue); + if (str != nullptr) { + const char* text = PyUnicode_AsUTF8(str); + if (text != nullptr) { + message = text; + } + Py_DECREF(str); + } + } + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + return message; +} + +// --------------------------------------------------------------------------- +// TypedDataArray -> NumPy (for custom-function inputs). Copying variant. +// --------------------------------------------------------------------------- + +void free_wrapped_memory(void* data) +{ + std::free(data); +} + +template +PyObject* wrap_array_copy(const libtokamap::TypedDataArray& data) +{ + const auto& shape = data.shape(); + int ndim = static_cast(shape.size()); + std::vector dims(shape.begin(), shape.end()); + + const size_t bytes = data.size() * sizeof(T); + // malloc(0) may legitimately return nullptr, so never ask for zero bytes — + // that keeps "nullptr means allocation failure" true for empty arrays too. + void* copied = std::malloc(bytes != 0 ? bytes : 1); + if (copied == nullptr) { + PyErr_NoMemory(); + return nullptr; + } + if (bytes != 0) { + std::memcpy(copied, data.data(), bytes); + } + + PyArray_Descr* descr = PyArray_DescrFromType(NPY_TYPE); + if (descr == nullptr) { + std::free(copied); + return nullptr; + } + PyObject* array = PyArray_NewFromDescr(&PyArray_Type, descr, ndim, dims.data(), nullptr, copied, + NPY_ARRAY_CARRAY, nullptr); + if (array == nullptr) { + std::free(copied); + return nullptr; + } + PyObject* capsule = PyCapsule_New(copied, nullptr, [](PyObject* cap) { std::free(PyCapsule_GetPointer(cap, nullptr)); }); + if (capsule == nullptr) { + Py_DECREF(array); + return nullptr; + } + PyArray_SetBaseObject(reinterpret_cast(array), capsule); + return array; +} + +PyObject* typed_data_array_to_numpy(const libtokamap::TypedDataArray& data) +{ + using libtokamap::DataType; + switch (data.data_type()) { + case DataType::Double: + return wrap_array_copy(data); + case DataType::Float: + return wrap_array_copy(data); + case DataType::Int64: + return wrap_array_copy(data); + case DataType::Int32: + return wrap_array_copy(data); + case DataType::Int16: + return wrap_array_copy(data); + case DataType::Int8: + return wrap_array_copy(data); + case DataType::UInt64: + return wrap_array_copy(data); + case DataType::UInt32: + return wrap_array_copy(data); + case DataType::UInt16: + return wrap_array_copy(data); + case DataType::UInt8: + return wrap_array_copy(data); + default: + Py_INCREF(Py_None); + return Py_None; + } +} + +// --------------------------------------------------------------------------- +// Embedded interpreter lifecycle — lazy, once per uda_server process +// --------------------------------------------------------------------------- + +std::once_flag g_python_once; + +// The thread state Py_Initialize() leaves current on the initialising thread, +// kept only so that the hand-off of the GIL is explicit and greppable. +PyThreadState* g_main_thread_state = nullptr; + +// RAII for the GIL. Re-entrant by design: on the thread that has just run +// Py_Initialize() the GIL is already held, so Ensure()/Release() are no-ops. +class GilLock +{ + public: + GilLock() : m_state{PyGILState_Ensure()} {} + ~GilLock() { PyGILState_Release(m_state); } + + GilLock(const GilLock&) = delete; + GilLock& operator=(const GilLock&) = delete; + GilLock(GilLock&&) = delete; + GilLock& operator=(GilLock&&) = delete; + + private: + PyGILState_STATE m_state; +}; + +// Py_Initialize() returns with the GIL held by the calling thread. Drop it on +// the way out of the initialisation — including via an exception — otherwise no +// Python thread can ever run while the plugin sits idle between requests, and a +// second thread calling PyGILState_Ensure() would block forever. +class MainThreadGilRelease +{ + public: + explicit MainThreadGilRelease(bool armed) : m_armed{armed} {} + ~MainThreadGilRelease() + { + if (m_armed) { + g_main_thread_state = PyEval_SaveThread(); + } + } + + MainThreadGilRelease(const MainThreadGilRelease&) = delete; + MainThreadGilRelease& operator=(const MainThreadGilRelease&) = delete; + MainThreadGilRelease(MainThreadGilRelease&&) = delete; + MainThreadGilRelease& operator=(MainThreadGilRelease&&) = delete; + + private: + bool m_armed; +}; + +// Prepends a colon-separated search path to sys.path, keeping the order of its +// entries the way CPython does it — leftmost entry wins. +void prepend_search_path(PyObject* sys_path, const char* search_path) +{ + if (search_path == nullptr || *search_path == '\0') { + return; + } + const std::string paths{search_path}; + Py_ssize_t index = 0; + size_t pos = 0; + while (pos <= paths.size()) { + const size_t sep = paths.find(':', pos); + const std::string entry = paths.substr(pos, sep == std::string::npos ? sep : sep - pos); + if (!entry.empty()) { + PyObject* pyentry = PyUnicode_FromString(entry.c_str()); + if (pyentry != nullptr) { + if (PyList_Insert(sys_path, index, pyentry) == 0) { + ++index; + } + Py_DECREF(pyentry); + } + } + if (sep == std::string::npos) { + break; + } + pos = sep + 1; + } +} + +void ensure_python_interpreter() +{ + std::call_once(g_python_once, [] { + bool initialised_here = false; + if (Py_IsInitialized() == 0) { + // Default the libpython path from the compile-time Python, but + // allow the deployment to override it. + const char* libpython = std::getenv("MAPPING_PLUGIN_LIBPYTHON"); + if (libpython == nullptr || *libpython == '\0') { + libpython = MAPPING_PLUGIN_DEFAULT_LIBPYTHON; + } + if (libpython != nullptr && *libpython != '\0') { + if (dlopen(libpython, RTLD_NOW | RTLD_GLOBAL) == nullptr) { + throw std::runtime_error{std::string{"failed to load libpython: "} + dlerror()}; + } + } + Py_Initialize(); // returns with the GIL held by this thread + if (Py_IsInitialized() == 0) { + throw std::runtime_error{"Py_Initialize failed"}; + } + initialised_here = true; + } + + // Declared first so it is destroyed last: the GIL taken by + // Py_Initialize() is handed back only once everything below is done. + MainThreadGilRelease main_thread_gil{initialised_here}; + // Everything below touches interpreter state, so it needs the GIL held. + // That includes the path where somebody else (the host process, another + // UDA plugin) initialised CPython and this thread holds nothing. + GilLock gil; + + if (initialised_here) { + // An embedded interpreter sees neither the caller's PYTHONPATH nor + // the venv paths, so fold both into sys.path explicitly. + PyObject* sys_path = PySys_GetObject("path"); // borrowed ref + if (sys_path != nullptr) { + prepend_search_path(sys_path, std::getenv("PYTHONPATH")); + // Applied second, so it takes precedence over PYTHONPATH. + prepend_search_path(sys_path, std::getenv("MAPPING_PLUGIN_PYTHONPATH")); + } + } + + if (_import_array() != 0) { + throw std::runtime_error{"NumPy C API initialisation failed: " + fetch_python_error()}; + } + }); +} + +// --------------------------------------------------------------------------- +// Python/C++ value conversions (mirrors clibtokamap) +// --------------------------------------------------------------------------- + +// Returns a new reference, or nullptr with the Python error indicator set. +PyObject* json_to_pyobject(const nlohmann::json& value) +{ + if (value.is_object()) { + PyObject* dict = PyDict_New(); + if (dict == nullptr) { + return nullptr; + } + for (auto& [key, sub] : value.items()) { + PyObject* py_sub = json_to_pyobject(sub); + if (py_sub == nullptr) { + Py_DECREF(dict); + return nullptr; + } + const int rc = PyDict_SetItemString(dict, key.c_str(), py_sub); + Py_DECREF(py_sub); + if (rc != 0) { + Py_DECREF(dict); + return nullptr; + } + } + return dict; + } + if (value.is_array()) { + PyObject* list = PyList_New(static_cast(value.size())); + if (list == nullptr) { + return nullptr; + } + for (size_t i = 0; i < value.size(); ++i) { + PyObject* item = json_to_pyobject(value[i]); + if (item == nullptr) { + Py_DECREF(list); // list dealloc copes with the unfilled slots + return nullptr; + } + PyList_SET_ITEM(list, static_cast(i), item); // steals + } + return list; + } + if (value.is_string()) { + return PyUnicode_FromString(value.get().c_str()); + } + if (value.is_boolean()) { + return PyBool_FromLong(value.get() ? 1 : 0); + } + if (value.is_number_integer()) { + return PyLong_FromLongLong(value.get()); + } + if (value.is_number_float()) { + return PyFloat_FromDouble(value.get()); + } + if (value.is_null()) { + Py_RETURN_NONE; + } + return PyUnicode_FromString(value.dump().c_str()); +} + +// Builds a Python dict out of any map of string -> nlohmann::json (the mapping +// arguments, a data source's constructor kwargs). Returns a new reference, or +// nullptr with the Python error indicator set. +template +PyObject* json_map_to_pydict(const JsonMap& values) +{ + PyObject* dict = PyDict_New(); + if (dict == nullptr) { + return nullptr; + } + for (const auto& [key, value] : values) { + PyObject* py_value = json_to_pyobject(value); + if (py_value == nullptr) { + Py_DECREF(dict); + return nullptr; + } + const int rc = PyDict_SetItemString(dict, key.c_str(), py_value); + Py_DECREF(py_value); + if (rc != 0) { + Py_DECREF(dict); + return nullptr; + } + } + return dict; +} + +bool numpy_to_typed_data_array(PyObject* object, libtokamap::TypedDataArray& out, std::string& error) +{ + if (PyUnicode_Check(object)) { + const char* text = PyUnicode_AsUTF8(object); + if (text == nullptr) { + error = "failed to convert Python str to UTF-8"; + return false; + } + out = libtokamap::TypedDataArray{std::string{text}}; + return true; + } + if (PyArray_Check(object) == 0) { + error = "Python data source must return a NumPy array or str"; + return false; + } + auto* array = reinterpret_cast(object); + // ISCARRAY_RO, not ISCARRAY: the data is copied below, so a read-only array + // (np.broadcast_to, an mmap view, a cache deliberately marked read-only) is + // perfectly usable and must not be rejected for not being writeable. + if (PyArray_ISCARRAY_RO(array) == 0) { + error = PyArray_IS_C_CONTIGUOUS(array) == 0 + ? "Python data source returned a non-C-contiguous NumPy array" + : "Python data source returned a misaligned NumPy array"; + return false; + } + void* data = PyArray_DATA(array); + auto size = static_cast(PyArray_SIZE(array)); + int rank = PyArray_NDIM(array); + npy_intp* shape = PyArray_DIMS(array); + std::vector shape_vec(shape, shape + rank); + + // The TypedDataArray(T*, size, shape) constructor copies the data, so the + // NumPy array can be released as soon as this call returns. + switch (PyArray_TYPE(array)) { + case NPY_BOOL: { + auto* bool_data = reinterpret_cast(data); + std::vector values(bool_data, bool_data + size); + out = libtokamap::TypedDataArray{values, shape_vec}; + return true; + } + case NPY_INT8: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_INT16: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_INT32: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_INT64: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_UINT8: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_UINT16: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_UINT32: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_UINT64: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_FLOAT32: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + case NPY_FLOAT64: + out = libtokamap::TypedDataArray{reinterpret_cast(data), size, shape_vec}; + return true; + default: + error = "Python data source returned data with unsupported dtype"; + return false; + } +} + +// --------------------------------------------------------------------------- +// PythonDataSource — a libtokamap::DataSource whose get() calls a Python +// object's .get(args) method. Mirrors libtokamap's own (clibtokamap) bridge. +// --------------------------------------------------------------------------- + +class PythonDataSource final : public libtokamap::DataSource +{ + public: + explicit PythonDataSource(PyObject* instance) : m_instance{instance} {} + + ~PythonDataSource() override + { + // uda_server exits after its connection so the interpreter is still up + // in practice, but take the GIL defensively for the decref. + if (Py_IsInitialized() != 0 && m_instance != nullptr) { + PyGILState_STATE gil = PyGILState_Ensure(); + Py_DECREF(m_instance); + PyGILState_Release(gil); + } + } + + PythonDataSource(PythonDataSource&&) = delete; + PythonDataSource& operator=(PythonDataSource&&) = delete; + PythonDataSource(const PythonDataSource&) = delete; + PythonDataSource& operator=(const PythonDataSource&) = delete; + + libtokamap::TypedDataArray get(const libtokamap::DataSourceArgs& map_args, + const libtokamap::MapArguments& /*arguments*/, + libtokamap::RamCache* /*ram_cache*/) override + { + PyGILState_STATE gil = PyGILState_Ensure(); + + libtokamap::TypedDataArray array; + std::string error; + + PyObject* args_dict = json_map_to_pydict(map_args); + if (args_dict == nullptr) { + error = "failed to build the arguments dict: " + fetch_python_error(); + } else { + PyObject* result = PyObject_CallMethod(m_instance, "get", "O", args_dict); + Py_DECREF(args_dict); + + if (result == nullptr) { + error = fetch_python_error(); + } else { + numpy_to_typed_data_array(result, array, error); + Py_DECREF(result); + } + } + PyGILState_Release(gil); + + if (!error.empty()) { + throw libtokamap::DataSourceError{"python data source get failed: " + error}; + } + return array; + } + + private: + PyObject* m_instance; +}; + +// --------------------------------------------------------------------------- +// PythonCustomFunction — wraps a Python callable as a libtokamap +// LibraryFunctionWrapper: call(inputs, params) -> function(inputs, params). +// --------------------------------------------------------------------------- + +class PythonCustomFunction final : public libtokamap::LibraryFunctionWrapper +{ + public: + explicit PythonCustomFunction(PyObject* function) : m_function{function} {} + + ~PythonCustomFunction() override + { + if (Py_IsInitialized() != 0 && m_function != nullptr) { + PyGILState_STATE gil = PyGILState_Ensure(); + Py_DECREF(m_function); + PyGILState_Release(gil); + } + } + + libtokamap::TypedDataArray operator()(libtokamap::CustomMappingInputs& inputs, + const libtokamap::CustomMappingParams& params) const override + { + PyGILState_STATE gil = PyGILState_Ensure(); + + libtokamap::TypedDataArray array; + std::string error; + + PyObject* inputs_dict = PyDict_New(); + if (inputs_dict == nullptr) { + error = "failed to allocate the inputs dict: " + fetch_python_error(); + } + for (auto& [key, value] : inputs) { + if (!error.empty()) { + break; + } + // A failed conversion returns nullptr; handing that to + // PyDict_SetItemString would dereference it. + PyObject* py_array = typed_data_array_to_numpy(value); + if (py_array == nullptr) { + error = "failed to convert input '" + key + "' to a NumPy array: " + fetch_python_error(); + break; + } + const int rc = PyDict_SetItemString(inputs_dict, key.c_str(), py_array); + Py_DECREF(py_array); + if (rc != 0) { + error = "failed to add input '" + key + "' to the inputs dict: " + fetch_python_error(); + break; + } + } + + if (error.empty()) { + PyObject* params_dict = json_to_pyobject(params); + PyObject* call_args = params_dict == nullptr ? nullptr : PyTuple_Pack(2, inputs_dict, params_dict); + Py_XDECREF(params_dict); + if (call_args == nullptr) { + error = "failed to build the call arguments: " + fetch_python_error(); + } else { + PyObject* result = PyObject_CallObject(m_function, call_args); + Py_DECREF(call_args); + + if (result == nullptr) { + error = fetch_python_error(); + } else { + numpy_to_typed_data_array(result, array, error); + Py_DECREF(result); + } + } + } + Py_XDECREF(inputs_dict); + PyGILState_Release(gil); + + if (!error.empty()) { + throw libtokamap::DataSourceError{"python custom function failed: " + error}; + } + return array; + } + + private: + PyObject* m_function; +}; + +// --------------------------------------------------------------------------- +// Config reading +// --------------------------------------------------------------------------- + +struct PythonDataSourceSpec +{ + std::string name; + std::string module; + std::string class_name; + std::unordered_map args; +}; + +struct PythonCustomFunctionSpec +{ + std::string library; + std::string module; + std::vector functions; +}; + +nlohmann::json load_python_config(const std::filesystem::path& config_path) +{ + std::ifstream stream{config_path}; + if (!stream) { + throw std::runtime_error{"failed to open MAPPING_PLUGIN_PYTHON_CONFIG: " + config_path.string()}; + } + + if (config_path.extension() == ".toml") { + try { + const auto toml_config = toml::parse(stream, config_path.string()); + std::stringstream json_stream; + json_stream << toml::json_formatter{toml_config}; + return nlohmann::json::parse(json_stream); + } catch (const toml::parse_error& e) { + throw std::runtime_error{"failed to parse " + config_path.string() + " as TOML: " + e.what()}; + } catch (const nlohmann::json::parse_error& e) { + throw std::runtime_error{"failed to convert " + config_path.string() + " from TOML to JSON: " + + e.what()}; + } + } + + if (config_path.extension() == ".json") { + try { + // Keep JSONC compatibility for deployments that use explanatory + // comments in the plugin config. + return nlohmann::json::parse(stream, nullptr, true, true); + } catch (const nlohmann::json::parse_error& e) { + throw std::runtime_error{"failed to parse " + config_path.string() + " as JSON: " + e.what()}; + } + } + + throw std::runtime_error{"unsupported MAPPING_PLUGIN_PYTHON_CONFIG extension for " + config_path.string() + + "; expected .toml or .json"}; +} + +// Reads a required non-empty string member. Returns false with `error` set when +// it is missing, not a string, or empty. +bool required_string(const nlohmann::json& object, const std::string& path, const char* key, std::string& out, + std::string& error) +{ + const auto member = object.find(key); + if (member == object.end() || !member->is_string() || member->get().empty()) { + error = path + " requires a non-empty '" + key + "' string"; + return false; + } + out = member->get(); + return true; +} + +bool parse_python_data_sources(const nlohmann::json& config, std::vector& specs, + std::string& error) +{ + const auto table = config.find("python_data_sources"); + if (table == config.end()) { + return true; // nothing configured + } + if (!table->is_object()) { + error = "python_data_sources must be an object"; + return false; + } + + for (const auto& [key, node] : table->items()) { + const std::string path = "python_data_sources." + key; + if (!node.is_object()) { + error = path + " must be an object"; + return false; + } + PythonDataSourceSpec spec; + spec.name = key; + if (!required_string(node, path, "module", spec.module, error)) { + return false; + } + // 'class_name', or the shorter 'class', defaulting to DataSource. + spec.class_name = spec.name + "DataSource"; + for (const char* alias : {"class", "class_name"}) { // class_name wins: checked last + const auto member = node.find(alias); + if (member == node.end()) { + continue; + } + if (!member->is_string() || member->get().empty()) { + error = path + "." + alias + " must be a non-empty string"; + return false; + } + spec.class_name = member->get(); + } + const auto args = node.find("args"); + if (args != node.end()) { + if (!args->is_object()) { + error = path + ".args must be an object"; + return false; + } + // Any JSON value goes through: json_to_pyobject renders nested + // objects and arrays as dicts and lists. + for (const auto& [arg_name, value] : args->items()) { + spec.args[arg_name] = value; + } + } + specs.push_back(std::move(spec)); + } + return true; +} + +bool parse_python_custom_functions(const nlohmann::json& config, std::vector& specs, + std::string& error) +{ + const auto table = config.find("python_custom_functions"); + if (table == config.end()) { + return true; // nothing configured + } + if (!table->is_object()) { + error = "python_custom_functions must be an object"; + return false; + } + + for (const auto& [key, node] : table->items()) { + const std::string path = "python_custom_functions." + key; + if (!node.is_object()) { + error = path + " must be an object"; + return false; + } + PythonCustomFunctionSpec spec; + spec.library = key; + if (!required_string(node, path, "module", spec.module, error)) { + return false; + } + const auto functions = node.find("functions"); + if (functions == node.end() || !functions->is_array()) { + error = path + " requires a 'functions' array"; + return false; + } + for (const auto& name : *functions) { + if (!name.is_string() || name.get().empty()) { + error = path + ".functions must be non-empty strings"; + return false; + } + spec.functions.push_back(name.get()); + } + specs.push_back(std::move(spec)); + } + return true; +} + +} // namespace + +void init_python_data_sources_if_configured(libtokamap::MappingHandler& mapping_handler) +{ + const char* config_env = std::getenv("MAPPING_PLUGIN_PYTHON_CONFIG"); + if (config_env == nullptr || *config_env == '\0') { + return; // no Python config declared anywhere — interpreter stays off + } + const std::filesystem::path config_path{config_env}; + if (!std::filesystem::exists(config_path)) { + throw std::runtime_error{"MAPPING_PLUGIN_PYTHON_CONFIG points to a missing file: " + config_path.string()}; + } + + const nlohmann::json config = load_python_config(config_path); + if (!config.is_object()) { + throw std::runtime_error{config_path.string() + " must contain a configuration object at the top level"}; + } + + std::vector source_specs; + std::vector function_specs; + std::string error; + if (!parse_python_data_sources(config, source_specs, error) || + !parse_python_custom_functions(config, function_specs, error)) { + throw std::runtime_error{error}; + } + if (source_specs.empty() && function_specs.empty()) { + return; // nothing Python-backed configured — interpreter stays off + } + + ensure_python_interpreter(); + + PyGILState_STATE gil = PyGILState_Ensure(); + for (const auto& spec : source_specs) { + PyObject* module = PyImport_ImportModule(spec.module.c_str()); + if (module == nullptr) { + std::string message = fetch_python_error(); + PyGILState_Release(gil); + throw std::runtime_error{"failed to import '" + spec.module + "': " + message}; + } + PyObject* cls = PyObject_GetAttrString(module, spec.class_name.c_str()); + Py_DECREF(module); + if (cls == nullptr) { + std::string message = fetch_python_error(); + PyGILState_Release(gil); + throw std::runtime_error{"failed to find class '" + spec.class_name + "' in " + spec.module + ": " + + message}; + } + PyObject* kwargs = json_map_to_pydict(spec.args); + if (kwargs == nullptr) { + std::string message = fetch_python_error(); + Py_DECREF(cls); + PyGILState_Release(gil); + throw std::runtime_error{"failed to build the arguments for " + spec.module + "." + spec.class_name + ": " + + message}; + } + PyObject* empty_args = PyTuple_New(0); + PyObject* instance = empty_args == nullptr ? nullptr : PyObject_Call(cls, empty_args, kwargs); + Py_XDECREF(empty_args); + Py_DECREF(kwargs); + Py_DECREF(cls); + if (instance == nullptr) { + std::string message = fetch_python_error(); + PyGILState_Release(gil); + throw std::runtime_error{"failed to construct " + spec.module + "." + spec.class_name + ": " + message}; + } + // The registry takes ownership; the Python object stays alive for the + // process lifetime via the reference held in PythonDataSource. + // + // Drop any previous registration of this name first: register_data_source + // throws on a duplicate, and this function can legitimately run again in + // the same process (a retry after a partially failed init). erase() is a + // no-op when the name is absent. + mapping_handler.unregister_data_source(spec.name); + mapping_handler.register_data_source(spec.name, std::make_unique(instance)); + } + + for (const auto& spec : function_specs) { + PyObject* module = PyImport_ImportModule(spec.module.c_str()); + if (module == nullptr) { + std::string message = fetch_python_error(); + PyGILState_Release(gil); + throw std::runtime_error{"failed to import '" + spec.module + "': " + message}; + } + for (const auto& function_name : spec.functions) { + PyObject* function = PyObject_GetAttrString(module, function_name.c_str()); + if (function == nullptr) { + std::string message = fetch_python_error(); + Py_DECREF(module); + PyGILState_Release(gil); + throw std::runtime_error{"failed to find function '" + function_name + "' in " + spec.module + ": " + + message}; + } + // As above: drop a previous registration so a repeated init replaces + // it instead of stacking a second entry that shadows the first. + // unregister_custom_function throws when there is nothing to drop. + try { + mapping_handler.unregister_custom_function(spec.library, function_name); + } catch (const std::exception&) { // not registered yet — nothing to drop + } + mapping_handler.register_custom_function(libtokamap::LibraryFunction{ + spec.library, function_name, std::make_unique(function)}); + } + Py_DECREF(module); + } + PyGILState_Release(gil); +} + +} // namespace mapping_plugin + +#else // !MAPPING_PLUGIN_PYTHON — stub: no Python dependency compiled in + +#include +#include + +namespace mapping_plugin +{ + +void init_python_data_sources_if_configured(libtokamap::MappingHandler& /*mapping_handler*/) +{ + // Configurations that declare Python data sources cannot run on a build + // without Python support — fail loudly rather than silently serve nothing. + const char* config_env = std::getenv("MAPPING_PLUGIN_PYTHON_CONFIG"); + if (config_env == nullptr || *config_env == '\0') { + return; + } + if (std::filesystem::exists(std::filesystem::path{config_env})) { + throw std::runtime_error{std::string{"MAPPING_PLUGIN_PYTHON_CONFIG is set ("} + config_env + + ") but this plugin was built without MAPPING_PLUGIN_PYTHON; " + "rebuild with -DMAPPING_PLUGIN_PYTHON=ON"}; + } +} + +} // namespace mapping_plugin + +#endif // MAPPING_PLUGIN_PYTHON diff --git a/mapping_plugin/src/python_data_source.hpp b/mapping_plugin/src/python_data_source.hpp new file mode 100644 index 0000000..b07d468 --- /dev/null +++ b/mapping_plugin/src/python_data_source.hpp @@ -0,0 +1,59 @@ +#ifndef MAPPING_PLUGIN_PYTHON_DATA_SOURCE_HPP +#define MAPPING_PLUGIN_PYTHON_DATA_SOURCE_HPP + +// Optional embedded-Python data-source support for the mapping plugin. +// +// This is compiled ONLY when the plugin is built with MAPPING_PLUGIN_PYTHON=ON +// (requires Python3 + NumPy headers). When off, the plugin has no Python +// dependency at all and init_python_data_sources_if_configured() is a no-op, +// so the plugin keeps working in environments that do not require Python. +// +// The interpreter is started lazily — only when a config file declares Python +// data sources or custom functions — and lives for +// the rest of the uda_server process (one connection per process under +// xinetd), so all fields served in one connection share the interpreter, +// the registered data-source instances and their caches. + +#include + +#include + +namespace mapping_plugin +{ + +// Read the plugin's python-data-source config and, if it declares any +// [python_data_sources] / [python_custom_functions], start the embedded +// interpreter (once per process), instantiate the configured classes and +// register the custom functions on mapping_handler. +// +// The declarations live in a SEPARATE TOML or JSON file pointed to by the env var +// MAPPING_PLUGIN_PYTHON_CONFIG — they cannot sit in the libtokamap config +// because libtokamap validates it against a built-in schema with +// additionalProperties=false. +// +// Config shape: +// [python_data_sources.CDB] +// module = "tokamap_compass.cdb_datasource" +// class_name = "CDBDataSource" # optional, default DataSource +// +// [python_data_sources.CDB.args] # optional constructor kwargs +// +// [python_custom_functions.compass] # library name +// module = "tokamap_compass.custom_functions" +// functions = ["interp1d", "zero_as_nan"] +// +// TOML parsing uses toml.hpp exported by libtokamap; the plugin does not vendor +// its own copy. JSON remains supported for compatibility (including // comments). +// Both formats are converted to nlohmann::json before the shared validation and +// registration path. +// +// No-op (interpreter never started) when the env var is unset, the file is +// absent, or both tables are empty. Throws std::runtime_error if Python +// sources are configured but could not be initialised. Builds without +// MAPPING_PLUGIN_PYTHON compile to a stub that never starts Python and only +// fails loudly if such a config is present. +void init_python_data_sources_if_configured(libtokamap::MappingHandler& mapping_handler); + +} // namespace mapping_plugin + +#endif // MAPPING_PLUGIN_PYTHON_DATA_SOURCE_HPP