diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aec5c55..8e8249db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -518,6 +518,80 @@ jobs: path: target/${{ matrix.target }}/release/${{ steps.binary_filename.outputs.name }} retention-days: 30 + # Build yang-consumer binary + # + # Windows and x86_64-apple-darwin are intentionally omitted: yang5/rdkafka + # don't build on Windows yet, and yang5 links system libpcre2 which can't be + # cross-compiled from arm64 macos-latest. + build-yang-consumer-bin: + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + needs: version + name: Build yang-consumer binary (${{ matrix.dist_and_arch }}) + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + binary_name: netcalyx-yang-consumer + strip_command: true + smoke_test: true + install_deps: | + sudo apt-get update + sudo apt-get install -y cmake perl gcc g++ libpcre2-dev libcurl4-openssl-dev + dist_and_arch: x86_64-ubuntu_latest # used just for naming the binary + - os: ubuntu-latest + container: rockylinux/rockylinux:8.10 + target: x86_64-unknown-linux-gnu + binary_name: netcalyx-yang-consumer + strip_command: true + smoke_test: true + install_deps: | + dnf install -y gcc gcc-c++ pcre2-devel perl-core cmake git curl libcurl-devel + dist_and_arch: x86_64-el8_10 # used just for naming the binary + - os: macos-latest + target: aarch64-apple-darwin + binary_name: netcalyx-yang-consumer + strip_command: true + smoke_test: true + dist_and_arch: aarch64-apple-latest # used just for naming the binary + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # needs full git history so shadow-rs embeds tag/commit info + - name: Install build dependencies + if: matrix.install_deps + run: ${{ matrix.install_deps }} + - name: Install stable Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + - name: Build yang-consumer + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + export C_INCLUDE_PATH="$C_INCLUDE_PATH:$HOMEBREW_PREFIX/include" + fi + cargo build --locked --release --target ${{ matrix.target }} -p netcalyx-yang-consumer + - name: Smoke test (run --help) + if: matrix.smoke_test == 'true' + run: ./target/${{ matrix.target }}/release/${{ matrix.binary_name }} --help + - name: Strip binary + if: matrix.strip_command == 'true' + run: strip target/${{ matrix.target }}/release/${{ matrix.binary_name }} + - name: Create complete binary file name + id: binary_filename + run: echo "name=netcalyx-yang-consumer-${{ needs.version.outputs.version }}-${{ needs.version.outputs.release }}-${{ matrix.dist_and_arch }}" >> "$GITHUB_OUTPUT" + - name: Rename binary + run: mv "target/${{ matrix.target }}/release/${{ matrix.binary_name }}" "target/${{ matrix.target }}/release/${{ steps.binary_filename.outputs.name }}" + - name: Upload yang-consumer artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ steps.binary_filename.outputs.name }} + path: target/${{ matrix.target }}/release/${{ steps.binary_filename.outputs.name }} + retention-days: 30 + commitlint: runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index 8c66e5ff..78c4b2d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2799,7 +2799,6 @@ dependencies = [ "tracing", "tracing-subscriber", "tracing-test", - "yang5", ] [[package]] @@ -3035,6 +3034,25 @@ dependencies = [ "tracing-test", ] +[[package]] +name = "netcalyx-yang-consumer" +version = "0.2.0" +dependencies = [ + "anyhow", + "clap", + "futures-util", + "netcalyx-netconf-proto", + "netcalyx-yang-push", + "rdkafka", + "schema-registry-client", + "serde_json", + "shadow-rs", + "tokio", + "tracing", + "tracing-subscriber", + "yang5", +] + [[package]] name = "netcalyx-yang-push" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 5964854f..669083d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/udp-notif-service", "crates/collector", "crates/yang-push", + "crates/yang-consumer", "crates/pcap-decoder", "crates/netconf-proto", "fuzz", @@ -40,6 +41,7 @@ netcalyx-udp-notif-pkt = { version = "0.2.0", path = "crates/udp-notif-pkt" } netcalyx-udp-notif-service = { version = "0.2.0", path = "crates/udp-notif-service" } netcalyx-analytics = { version = "0.2.0", path = "crates/analytics" } netcalyx-yang-push = { version = "0.2.0", path = "crates/yang-push" } +netcalyx-yang-consumer = { version = "0.2.0", path = "crates/yang-consumer" } netcalyx-pcap-decoder = { version = "0.2.0", path = "crates/pcap-decoder" } netcalyx-netconf-proto = { version = "0.2.0", path = "crates/netconf-proto" } thiserror = "2.0" diff --git a/crates/collector/Cargo.toml b/crates/collector/Cargo.toml index a88c0ac4..86fa850c 100644 --- a/crates/collector/Cargo.toml +++ b/crates/collector/Cargo.toml @@ -125,7 +125,6 @@ chrono = { workspace = true, default-features = false, features = [ "clock", ] } tempfile = { workspace = true } -yang5 = { workspace = true } tracing-test = { workspace = true } [features] diff --git a/crates/yang-consumer/Cargo.toml b/crates/yang-consumer/Cargo.toml new file mode 100644 index 00000000..e3f01fac --- /dev/null +++ b/crates/yang-consumer/Cargo.toml @@ -0,0 +1,43 @@ +lints.workspace = true + +[package] +name = "netcalyx-yang-consumer" +version = "0.2.0" +edition = "2024" +authors = ["NetCalyx Authors"] +license = "Apache-2.0" +readme = "README.md" +repository = "https://github.com/network-analytics/NetCalyx" +homepage = "https://github.com/network-analytics/NetCalyx" +description = """ +CLI utility to consume YANG-encoded telemetry messages from Kafka and validate them against YANG schemas from a Schema Registry. +""" +keywords = ["yang", "kafka", "telemetry", "schema-registry", "validation"] +categories = ["network-programming", "command-line-utilities"] +build = "build.rs" + +[[bin]] +name = "netcalyx-yang-consumer" +path = "src/main.rs" + +[dependencies] +netcalyx-netconf-proto = { workspace = true } +netcalyx-yang-push = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +futures-util = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true, default-features = false, features = ["net", "io-util", "sync", "time", "rt-multi-thread", "macros", "signal"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +shadow-rs = { workspace = true, features = ["default", "metadata", "build"] } +yang5 = { workspace = true } +schema-registry-client = { workspace = true } +rdkafka = { workspace = true, features = [ + "tokio", + "cmake-build", + "ssl-vendored", +] } + +[build-dependencies] +shadow-rs = { workspace = true, default-features = true } diff --git a/crates/yang/NOTICE b/crates/yang-consumer/NOTICE similarity index 100% rename from crates/yang/NOTICE rename to crates/yang-consumer/NOTICE diff --git a/crates/yang-consumer/README.md b/crates/yang-consumer/README.md new file mode 100644 index 00000000..e095a3f0 --- /dev/null +++ b/crates/yang-consumer/README.md @@ -0,0 +1,94 @@ +# NetCalyx YANG Consumer + +A CLI utility that consumes YANG-encoded telemetry messages from Kafka +and validates them against YANG schemas stored in a Schema Registry. + +## Features + +- Consumes messages from a Kafka topic with configurable partitions and offsets +- Fetches YANG schemas (including dependencies) from a Schema Registry +- Builds and caches YANG Library contexts for validation +- Validates message payloads against their associated YANG schemas +- Supports tail mode to read the last N messages per partition +- Supports follow mode to continuously consume new messages + +## Installation + +```bash +cargo install netcalyx-yang-consumer +``` + +Or build from source: + +```bash +git clone https://github.com/network-analytics/NetCalyx.git +cd NetCalyx +cargo build --release -p netcalyx-yang-consumer +``` + +## CLI Usage + +```bash +netcalyx-yang-consumer --help +``` + +Logging is controlled via the `RUST_LOG` environment variable (defaults to `info`). + +### Examples + +Follow a topic on a local plaintext broker: + +```bash +RUST_LOG=netcalyx_yang_consumer=info,info \ + ./target/debug/netcalyx-yang-consumer \ + -b localhost:9092 \ + -s http://localhost:8081 \ + --group my-consumer-group \ + -t telemetry-message-yang \ + -f +``` + +Read the last 1000 messages using a librdkafka config file (e.g. for an +authenticated broker) and a remote Schema Registry: + +```bash +RUST_LOG=netcalyx_yang_consumer=info,info \ + ./target/debug/netcalyx-yang-consumer \ + -c librdkafka.json \ + -s http://schema-registry.example.com/ \ + --group my-consumer-group \ + -t device-yang-raw \ + -n 1000 +``` + +### librdkafka config files + +Any [librdkafka configuration property](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md) +can be passed via `--config-file` as a flat JSON object. + +CLI flags +(`--bootstrap-servers`, `--group`) take precedence over values set here. + +SASL over SSL (e.g. AWS MSK): + +```json +{ + "security.protocol": "SASL_SSL", + "sasl.mechanism": "SCRAM-SHA-512", + "sasl.username": "", + "sasl.password": "", + "metadata.broker.list": "broker-1.example.com:9096,broker-2.example.com:9096" +} +``` + +Mutual TLS with client certificates: + +```json +{ + "security.protocol": "ssl", + "ssl.certificate.location": "/path/to/cert.crt", + "ssl.key.location": "/path/to/client.key", + "ssl.ca.location": "/path/to/root_ca.crt", + "metadata.broker.list": "kafka.example.com:9093" +} +``` diff --git a/crates/yang-consumer/build.rs b/crates/yang-consumer/build.rs new file mode 100644 index 00000000..23c7c4ac --- /dev/null +++ b/crates/yang-consumer/build.rs @@ -0,0 +1,23 @@ +// Copyright (C) 2026-present The NetCalyx Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use shadow_rs::ShadowBuilder; + +fn main() { + ShadowBuilder::builder() + .deny_const(Default::default()) + .build() + .unwrap(); +} diff --git a/crates/collector/examples/kafka-yang-consumer.rs b/crates/yang-consumer/src/main.rs similarity index 99% rename from crates/collector/examples/kafka-yang-consumer.rs rename to crates/yang-consumer/src/main.rs index 0928a01c..32d26c70 100644 --- a/crates/collector/examples/kafka-yang-consumer.rs +++ b/crates/yang-consumer/src/main.rs @@ -62,7 +62,10 @@ shadow!(build); #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { - /// Librdkafka config file (json) + /// Librdkafka config file (json). + /// + /// Entries are passed directly to librdkafka as key/value properties, see: + /// #[arg(short = 'c', long)] config_file: Option,