From a0fe3d386fe8627580c7c00dfedb71ffac6e5f79 Mon Sep 17 00:00:00 2001 From: Tim Evens Date: Wed, 8 Jul 2026 17:23:45 -0700 Subject: [PATCH] Add relay health check program an dockerfile --- Makefile | 39 +++- healthcheck.Dockerfile | 56 ++++++ scripts/relay_health_http.py | 105 ++++++++++ src/CMakeLists.txt | 14 ++ src/relay_health_check.cpp | 370 +++++++++++++++++++++++++++++++++++ 5 files changed, 581 insertions(+), 3 deletions(-) create mode 100644 healthcheck.Dockerfile create mode 100644 scripts/relay_health_http.py create mode 100644 src/relay_health_check.cpp diff --git a/Makefile b/Makefile index fabc5437..8a6be201 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,12 @@ BUILD_JOBS?=4 BUILD_DIR?=build ECR_NAME?=laps-relay +HEALTHCHECK_ECR_NAME?=healthcheck CLANG_FORMAT=clang-format -i PROJECTNAME := laps -.PHONY: all clean cclean format docs +.PHONY: all clean cclean format docs image-healthcheck image-healthcheck-amd64 image-healthcheck-arm64 publish-healthcheck publish-healthcheck-amd64 publish-healthcheck-arm64 # ----------------------------------------- # Help/other targets @@ -73,7 +74,7 @@ docker-prep: @echo "Prep normally requires submodule update, but skipping considering possible custom changes" # @git submodule update --init --recursive -## image-amd64: Create AMD64 docker image∂ +## image-amd64: Create AMD64 docker image image-amd64: docker-prep @docker buildx build --progress=plain \ --output type=docker --platform linux/amd64 \ @@ -110,6 +111,21 @@ image-arm64: docker-prep --output type=docker --platform linux/arm64 \ -f Dockerfile -t quicr/${ECR_NAME}:${DOCKER_TAG}-arm64 . +## image-healthcheck: Create AMD64 and ARM64 relay health-check docker images +image-healthcheck: image-healthcheck-amd64 image-healthcheck-arm64 + +## image-healthcheck-amd64: Create AMD64 relay health-check docker image +image-healthcheck-amd64: docker-prep + @docker buildx build --progress=plain \ + --output type=docker --platform linux/amd64 \ + -f healthcheck.Dockerfile -t quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-amd64 . + +## image-healthcheck-arm64: Create ARM64 relay health-check docker image +image-healthcheck-arm64: docker-prep + @docker buildx build --progress=plain \ + --output type=docker --platform linux/arm64 \ + -f healthcheck.Dockerfile -t quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-arm64 . + ecr-login: @echo "==> Logging into ECR using environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY" @docker run --rm \ @@ -134,4 +150,21 @@ publish-image-arm64: ecr-login @echo "==> Pushing image 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${ECR_NAME}:${DOCKER_TAG}-arm64 to ECR" @docker push 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${ECR_NAME}:${DOCKER_TAG}-arm64 - +## publish-healthcheck: Publish amd64 and arm64 relay health-check images to ECR +publish-healthcheck: publish-healthcheck-amd64 publish-healthcheck-arm64 + +## publish-healthcheck-amd64: Publish amd64 relay health-check image to ECR +publish-healthcheck-amd64: ecr-login + @echo "==> Tagging docker image to 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-amd64" + @docker tag quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-amd64 \ + 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-amd64 + @echo "==> Pushing image 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-amd64 to ECR" + @docker push 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-amd64 + +## publish-healthcheck-arm64: Publish arm64 relay health-check image to ECR +publish-healthcheck-arm64: ecr-login + @echo "==> Tagging docker image to 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-arm64" + @docker tag quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-arm64 \ + 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-arm64 + @echo "==> Pushing image 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-arm64 to ECR" + @docker push 017125485914.dkr.ecr.us-west-1.amazonaws.com/quicr/${HEALTHCHECK_ECR_NAME}:${DOCKER_TAG}-arm64 diff --git a/healthcheck.Dockerfile b/healthcheck.Dockerfile new file mode 100644 index 00000000..a936140a --- /dev/null +++ b/healthcheck.Dockerfile @@ -0,0 +1,56 @@ +#--------------------------------------------------------------------- +# LAPS Relay health-check HTTP image +#--------------------------------------------------------------------- + +FROM alpine:3.20.3 AS builder + +RUN apk add --no-cache \ + alpine-sdk \ + bash \ + ca-certificates \ + clang \ + cmake \ + curl \ + linux-headers \ + lld \ + openssl-dev \ + python3 \ + tcsh + +WORKDIR /ws + +COPY ./CMakeLists.txt ./ +COPY ./version_config.h.in ./ +COPY ./dependencies ./dependencies +COPY ./src ./src + +ENV CFLAGS="-Wno-error=stringop-overflow" +ENV CXXFLAGS="-Wno-error=stringop-overflow -fpermissive -Wno-error=pedantic" + +RUN cmake -S . -B build \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=OFF \ + -DLAPS_BUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release +RUN cmake --build build --target relay_health_check -j "$(nproc)" +RUN cp build/src/relay_health_check /usr/local/bin/. + +FROM alpine:3.20.3 + +RUN apk add --no-cache libstdc++ python3 + +COPY --from=builder /usr/local/bin/relay_health_check /usr/local/bin/. +COPY ./scripts/relay_health_http.py /usr/local/bin/relay_health_http.py + +RUN addgroup -S laps +RUN adduser -D -S -S -G laps laps + +USER laps +WORKDIR /home/laps + +EXPOSE 8080/tcp + +ENV RELAY_HEALTH_HTTP_HOST=0.0.0.0 +ENV RELAY_HEALTH_HTTP_PORT=8080 + +CMD ["python3", "/usr/local/bin/relay_health_http.py"] diff --git a/scripts/relay_health_http.py b/scripts/relay_health_http.py new file mode 100644 index 00000000..997a98f1 --- /dev/null +++ b/scripts/relay_health_http.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco Systems +# SPDX-License-Identifier: BSD-2-Clause + +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import os +import subprocess + + +CHECK_BIN = os.environ.get("RELAY_HEALTH_CHECK_BIN", "/usr/local/bin/relay_health_check") +HTTP_HOST = os.environ.get("RELAY_HEALTH_HTTP_HOST", "0.0.0.0") +HTTP_PORT = int(os.environ.get("RELAY_HEALTH_HTTP_PORT", "8080")) +CHECK_TIMEOUT_SECONDS = float(os.environ.get("RELAY_HEALTH_HTTP_TIMEOUT_SECONDS", "10")) + + +def _to_text(value): + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value + + +def _detail_text(stdout, stderr, returncode): + chunks = [] + + stdout = _to_text(stdout) + stderr = _to_text(stderr) + + stdout_lines = stdout.strip().splitlines() + if stdout_lines and stdout_lines[0] in ("ok", "error"): + stdout_lines = stdout_lines[1:] + + stdout_details = "\n".join(stdout_lines).strip() + stderr_details = stderr.strip() + + if stdout_details: + chunks.append(stdout_details) + if stderr_details: + chunks.append(stderr_details) + + if not chunks: + chunks.append(f"relay_health_check exited with status {returncode}") + + return "\n".join(chunks) + + +def run_health_check(): + try: + result = subprocess.run( + [CHECK_BIN], + capture_output=True, + check=False, + text=True, + timeout=CHECK_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + details = _detail_text(stdout, stderr, "timeout") + details = f"relay_health_check timed out after {CHECK_TIMEOUT_SECONDS:g} seconds\n{details}" + return HTTPStatus.SERVICE_UNAVAILABLE, f"error\n\n{details.strip()}\n" + except OSError as exc: + return HTTPStatus.SERVICE_UNAVAILABLE, f"error\n\nfailed to run {CHECK_BIN}: {exc}\n" + + if result.returncode == 0: + return HTTPStatus.OK, "ok\n" + + details = _detail_text(result.stdout, result.stderr, result.returncode) + return HTTPStatus.SERVICE_UNAVAILABLE, f"error\n\n{details.strip()}\n" + + +class RelayHealthHandler(BaseHTTPRequestHandler): + server_version = "relay-health-http/1.0" + + def do_GET(self): + status, body = run_health_check() + body_bytes = body.encode("utf-8") + + self.send_response(status) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body_bytes))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body_bytes) + + def do_HEAD(self): + status, body = run_health_check() + body_bytes = body.encode("utf-8") + + self.send_response(status) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body_bytes))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + + def log_message(self, format, *args): + print(f"{self.address_string()} - {format % args}", flush=True) + + +if __name__ == "__main__": + server = ThreadingHTTPServer((HTTP_HOST, HTTP_PORT), RelayHealthHandler) + print(f"relay health HTTP server listening on {HTTP_HOST}:{HTTP_PORT}", flush=True) + server.serve_forever() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 09b2ff6a..73f97542 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -43,3 +43,17 @@ set_target_properties(lapsRelay CXX_EXTENSIONS ON) target_compile_definitions(lapsRelay PRIVATE SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG) + +add_executable(relay_health_check relay_health_check.cpp) +target_link_libraries(relay_health_check PRIVATE quicr) + +target_compile_options(relay_health_check + PRIVATE + $<$,$,$>: -Wpedantic -Wextra -Wall> + $<$: >) + +set_target_properties(relay_health_check + PROPERTIES + CXX_STANDARD 20 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS OFF) \ No newline at end of file diff --git a/src/relay_health_check.cpp b/src/relay_health_check.cpp new file mode 100644 index 00000000..608e5177 --- /dev/null +++ b/src/relay_health_check.cpp @@ -0,0 +1,370 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Cisco Systems +// SPDX-License-Identifier: BSD-2-Clause + +#include "quicr/client.h" +#include "quicr/publish_track_handler.h" +#include "quicr/subscribe_track_handler.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace quicr; + +namespace { + + struct Options + { + std::string uri{ "moq://laps-relay:12345/relay" }; + std::chrono::milliseconds timeout{ 5000 }; + std::string name_space{ "libquicr/health" }; + std::optional name; + std::string message{ "libquicr relay health check" }; + bool debug{ false }; + }; + + struct VerificationResult + { + bool matched{ false }; + std::string error; + }; + + template + bool WaitFor(Predicate predicate, + std::chrono::milliseconds timeout, + std::chrono::milliseconds poll_interval = std::chrono::milliseconds(10)) + { + const auto start = std::chrono::steady_clock::now(); + while (std::chrono::duration_cast(std::chrono::steady_clock::now() - start) < + timeout) { + if (predicate()) { + return true; + } + std::this_thread::sleep_for(poll_interval); + } + return predicate(); + } + + std::string GetEnvOrDefault(const char* key, std::string fallback) + { + const auto value = std::getenv(key); + if (value == nullptr || std::string_view(value).empty()) { + return fallback; + } + return value; + } + + std::chrono::milliseconds ParseTimeout(std::string_view value) + { + try { + return std::chrono::milliseconds(std::stoul(std::string(value))); + } catch (const std::exception&) { + throw std::runtime_error("timeout must be an integer number of milliseconds"); + } + } + + std::vector SplitNamespace(std::string_view value) + { + std::vector parts; + std::string current; + for (const char ch : value) { + if (ch == '/' || ch == ',') { + if (!current.empty()) { + parts.push_back(current); + current.clear(); + } + continue; + } + current.push_back(ch); + } + + if (!current.empty()) { + parts.push_back(current); + } + + if (parts.empty()) { + parts.emplace_back("health"); + } + + return parts; + } + + Bytes ToBytes(std::string_view value) + { + Bytes bytes; + bytes.reserve(value.size()); + for (const auto ch : value) { + bytes.push_back(static_cast(ch)); + } + return bytes; + } + + std::string MakeDefaultTrackName() + { + const auto now = std::chrono::steady_clock::now().time_since_epoch().count(); + return "probe-" + std::to_string(now); + } + + void ApplyEnvironment(Options& options) + { + options.uri = GetEnvOrDefault("LIBQUICR_RELAY_HEALTH_URI", options.uri); + options.name_space = GetEnvOrDefault("LIBQUICR_RELAY_HEALTH_NAMESPACE", options.name_space); + options.message = GetEnvOrDefault("LIBQUICR_RELAY_HEALTH_MESSAGE", options.message); + + if (const auto name = std::getenv("LIBQUICR_RELAY_HEALTH_NAME"); name != nullptr && *name != '\0') { + options.name = name; + } + + if (const auto timeout = std::getenv("LIBQUICR_RELAY_HEALTH_TIMEOUT_MS"); + timeout != nullptr && *timeout != '\0') { + options.timeout = ParseTimeout(timeout); + } + + if (const auto debug = std::getenv("LIBQUICR_RELAY_HEALTH_DEBUG"); debug != nullptr) { + options.debug = std::string_view(debug) == "1" || std::string_view(debug) == "true"; + } + } + + void PrintUsage(const char* program) + { + std::cerr << "Usage: " << program + << " [--uri URI] [--timeout-ms MS] [--namespace NS] [--name NAME] [--message TEXT] [--debug]\n" + << "\n" + << "Environment overrides:\n" + << " LIBQUICR_RELAY_HEALTH_URI\n" + << " LIBQUICR_RELAY_HEALTH_TIMEOUT_MS\n" + << " LIBQUICR_RELAY_HEALTH_NAMESPACE\n" + << " LIBQUICR_RELAY_HEALTH_NAME\n" + << " LIBQUICR_RELAY_HEALTH_MESSAGE\n" + << " LIBQUICR_RELAY_HEALTH_DEBUG\n"; + } + + Options ParseOptions(int argc, char* argv[]) + { + Options options; + ApplyEnvironment(options); + + auto require_value = [&](int& index, const std::string_view option) -> std::string { + if (index + 1 >= argc) { + throw std::runtime_error(std::string(option) + " requires a value"); + } + ++index; + return argv[index]; + }; + + for (int i = 1; i < argc; ++i) { + const std::string_view arg(argv[i]); + if (arg == "--help" || arg == "-h") { + PrintUsage(argv[0]); + std::exit(EXIT_SUCCESS); + } + if (arg == "--uri") { + options.uri = require_value(i, arg); + } else if (arg == "--timeout-ms") { + options.timeout = ParseTimeout(require_value(i, arg)); + } else if (arg == "--namespace") { + options.name_space = require_value(i, arg); + } else if (arg == "--name") { + options.name = require_value(i, arg); + } else if (arg == "--message") { + options.message = require_value(i, arg); + } else if (arg == "--debug") { + options.debug = true; + } else { + throw std::runtime_error("unknown option: " + std::string(arg)); + } + } + + return options; + } + + class VerifyingSubscribeTrackHandler final : public SubscribeTrackHandler + { + public: + static std::shared_ptr Create( + const FullTrackName& full_track_name, + Bytes expected_payload, + std::shared_ptr> result) + { + return std::shared_ptr( + new VerifyingSubscribeTrackHandler(full_track_name, std::move(expected_payload), std::move(result))); + } + + void ObjectReceived(const ObjectHeaders& object_headers, + BytesSpan data, + std::optional) override + { + std::lock_guard lock(mutex_); + if (completed_) { + return; + } + completed_ = true; + + const Bytes received(data.begin(), data.end()); + if (received != expected_payload_) { + std::ostringstream error; + error << "payload mismatch for group " << object_headers.group_id << ", subgroup " + << object_headers.subgroup_id << ", object " << object_headers.object_id << ": expected " + << expected_payload_.size() << " bytes, received " << received.size() << " bytes"; + result_->set_value({ .matched = false, .error = error.str() }); + return; + } + + result_->set_value({ .matched = true, .error = {} }); + } + + private: + VerifyingSubscribeTrackHandler(const FullTrackName& full_track_name, + Bytes expected_payload, + std::shared_ptr> result) + : SubscribeTrackHandler(full_track_name, 3, std::nullopt) + , expected_payload_(std::move(expected_payload)) + , result_(std::move(result)) + { + } + + std::mutex mutex_; + bool completed_{ false }; + Bytes expected_payload_; + std::shared_ptr> result_; + }; + + std::shared_ptr MakeClient(const std::string& endpoint_id, const Options& options) + { + ClientConfig config; + config.endpoint_id = endpoint_id; + config.connect_uri = options.uri; + config.transport_config.debug = options.debug; + config.transport_config.time_queue_max_duration = 10000; + config.transport_config.idle_timeout_ms = static_cast(options.timeout.count() * 2); + return Client::Create(config); + } + + bool RunHealthCheck(const Options& options, std::string& error) + { + const auto unique_name = options.name.value_or(MakeDefaultTrackName()); + const auto unique_suffix = unique_name; + const auto expected_payload = ToBytes(options.message); + + FullTrackName track; + track.name_space = TrackNamespace(SplitNamespace(options.name_space)); + track.name = ToBytes(unique_name); + + auto subscriber = MakeClient("relay-health-subscriber-" + unique_suffix, options); + auto publisher = MakeClient("relay-health-publisher-" + unique_suffix, options); + + subscriber->Connect(); + publisher->Connect(); + + const bool connected = WaitFor( + [&subscriber, &publisher]() { + return subscriber->GetStatus() == Transport::Status::kReady && + publisher->GetStatus() == Transport::Status::kReady; + }, + options.timeout); + if (!connected) { + error = "publisher and subscriber did not both connect before timeout"; + subscriber->Disconnect(); + publisher->Disconnect(); + return false; + } + + const auto result_promise = std::make_shared>(); + auto result_future = result_promise->get_future(); + auto sub_handler = VerifyingSubscribeTrackHandler::Create(track, expected_payload, result_promise); + auto pub_handler = PublishTrackHandler::Create(track, TrackMode::kStream, 3, 1000, { 0, 0 }); + + subscriber->SubscribeTrack(sub_handler); + publisher->PublishTrack(pub_handler); + + const bool ready = WaitFor( + [&sub_handler, &pub_handler]() { + return sub_handler->GetStatus() == SubscribeTrackHandler::Status::kOk && pub_handler->CanPublish(); + }, + options.timeout); + if (!ready) { + std::ostringstream stream; + stream << "publisher/subscriber track setup timed out; subscriber status=" + << static_cast(sub_handler->GetStatus()) + << ", publisher status=" << static_cast(pub_handler->GetStatus()); + error = stream.str(); + subscriber->Disconnect(); + publisher->Disconnect(); + return false; + } + + ObjectHeaders headers{ .group_id = 0, + .object_id = 0, + .subgroup_id = 0, + .payload_length = expected_payload.size(), + .status = ObjectStatus::kAvailable, + .priority = 3, + .ttl = 1000, + .track_mode = TrackMode::kStream, + .extensions = std::nullopt, + .immutable_extensions = std::nullopt }; + + const auto publish_status = pub_handler->PublishObject(headers, expected_payload); + if (publish_status != PublishTrackHandler::PublishObjectStatus::kOk) { + error = "PublishObject failed with status " + std::to_string(static_cast(publish_status)); + subscriber->Disconnect(); + publisher->Disconnect(); + return false; + } + + if (result_future.wait_for(options.timeout) != std::future_status::ready) { + error = "subscriber did not receive the health-check object before timeout"; + subscriber->Disconnect(); + publisher->Disconnect(); + return false; + } + + const auto result = result_future.get(); + subscriber->Disconnect(); + publisher->Disconnect(); + + if (!result.matched) { + error = result.error; + return false; + } + + return true; + } +} + +int +main(int argc, char* argv[]) +{ + try { + const auto options = ParseOptions(argc, argv); + spdlog::set_level(options.debug ? spdlog::level::debug : spdlog::level::off); + + std::string error; + if (RunHealthCheck(options, error)) { + std::cout << "ok\n"; + return EXIT_SUCCESS; + } + + std::cout << "error\n"; + if (!error.empty()) { + std::cerr << error << "\n"; + } + return EXIT_FAILURE; + } catch (const std::exception& e) { + std::cout << "error\n"; + std::cerr << e.what() << "\n"; + return EXIT_FAILURE; + } +}