Add suricata language server support (Dalton 5.0.0)

.gitignore

@@ -46,6 +46,8 @@ downloads/
 eggs/
 .eggs/
 lib/
+# vendored CodeMirror, not a Python build artifact
+!app/static/codemirror/lib/
 lib64/
 parts/
 sdist/

CHANGELOG.rst

@@ -2,6 +2,33 @@
 Dalton Changelog
 ****************
 
+5.0.0 (2026-09-17)
+##################
+
+* **Upgrade note:** the default ``docker-compose.yml`` now includes a new
+  ``linter`` service (container ``dalton_linter``, image ``dalton-linter``),
+  built from the ``linter`` target of ``Dockerfile_suricata``, so a
+  ``docker compose up`` after upgrading builds and starts one more container.
+  It runs the GPL-3.0 Suricata Language Server as a separate process (see the
+  "Suricata Rule Editor" section of the README for the licensing note). It is
+  optional: remove the service and leave ``rule_check_url`` / ``keywords_url``
+  empty in ``dalton.conf`` to run without it
+* Added an optional CodeMirror-based rule editor to the Suricata coverage page
+  (off by default; tick "Rule editor" next to the custom-rules box) with syntax
+  highlighting, keyword completion and hover documentation, and syntax checking
+  against a real Suricata engine
+* Added a ``linter`` sidecar container (built from the ``linter`` target of
+  ``Dockerfile_suricata``, sharing the Suricata compile) that runs the
+  `Suricata Language Server <https://github.com/StamusNetworks/suricata-language-server>`__
+  as a separate process; the controller reaches it only over HTTP and fails open
+  if it is unavailable, so job submission never depends on it
+* Added ``rule_check_url``, ``rule_check_timeout`` and ``keywords_url`` settings
+  to ``dalton.conf``; leave the URLs empty or absent to disable the feature
+* Added an optional "Engine analysis" toggle (on by default when the editor is
+  enabled) that includes Suricata's own performance and coverage guidance in
+  the syntax check results
+* Brought the ``VERSION`` file back in line with ``pyproject.toml``
+
 4.0.0 (2026-02-18)
 ##################
 

README.rst

@@ -73,6 +73,7 @@ Contents
 -  `Flowsynth WebUI <#flowsynth-webui>`__
 -  `Zeek <#zeek>`__
 -  `CyberChef <#cyberchef>`__
+-  `Suricata Rule Editor <#suricata-rule-editor>`__
 -  `Frequently Asked Questions <#frequently-asked-questions>`__
 -  `Authors <#authors>`__
 
@@ -1346,6 +1347,48 @@ For convenience, Dalton has the ability to easily build and run a
 in the Dalton toolbar, or directly using the '/cyberchef' URI path.
 
 
+Suricata Rule Editor
+=====================
+
+The custom-rules box on the Suricata coverage page has an optional rule
+editor: syntax highlighting, keyword completion with hover documentation,
+and real syntax checking against a running Suricata engine. It is off by
+default -- tick the "Rule editor" checkbox next to the custom-rules box to
+turn it on; the plain textarea remains the default experience and stays
+available underneath. A second checkbox, "Engine analysis" (on by default
+once the editor itself is enabled), adds Suricata's own performance and
+coverage guidance (e.g. "consider adding a flow keyword") to the syntax
+check.
+
+This is backed by a new ``linter`` container, built by default alongside
+the Suricata agents (it shares their Suricata compile via a build target in
+``dalton-agent/Dockerfiles/Dockerfile_suricata``) and reached by the
+controller over HTTP. Keyword data is generated from the linter's own
+Suricata engine at startup, not from any third-party database, so it is
+Apache-2.0-clean and always matches the engine actually being checked
+against.
+
+Syntax checking uses the `Suricata Language Server
+<https://github.com/StamusNetworks/suricata-language-server>`__ (SLS),
+which is GPL-3.0-licensed. SLS runs only inside the ``linter`` container,
+where it is invoked as a separate OS process via its own ``--batch-file``
+CLI (see ``dalton-agent/linter/app.py``) -- never imported into or linked
+with the linter's own (Apache-2.0) Flask app, so the two stay arm's-length
+programs communicating over an argv/stdout boundary. The (Apache-2.0)
+controller, in turn, never talks to SLS directly at all; it only ever
+reaches the ``linter`` container over HTTP.
+
+The feature is a convenience, not a dependency: if the ``linter`` container
+is unreachable, unhealthy, or disabled, the controller fails open and the
+coverage page simply loses syntax highlighting/checking, with everything
+else -- including job submission -- unaffected. To disable it entirely,
+remove, comment out, or leave empty the ``rule_check_url`` / ``keywords_url``
+settings in ``dalton.conf`` (an absent or empty URL means "off"; there is no
+built-in default); because ``dalton.conf`` is baked into the controller image
+at build time (see ``Dockerfile-dalton``), this requires rebuilding the
+controller image, not just restarting it.
+
+
 Frequently Asked Questions
 ==========================
 

VERSION

@@ -1 +1 @@
-3.6.0
+5.0.0

app/__init__.py

@@ -6,7 +6,7 @@
 from app.dalton import dalton_blueprint, ensure_rulesets_exist, setup_dalton_logging
 from app.flowsynth import flowsynth_blueprint, setup_flowsynth_logging
 
-__version__ = "4.0.0"
+__version__ = "5.0.0"
 
 
 def create_app(test_config=None):

app/dalton.py

@@ -38,13 +38,22 @@
 import tempfile
 import time
 import traceback
+import urllib.request
 import zipfile
 from distutils.version import LooseVersion
 from functools import lru_cache, wraps
 from logging.handlers import RotatingFileHandler
 from threading import Thread
 
-from flask import Blueprint, Response, redirect, render_template, request, url_for
+from flask import (
+    Blueprint,
+    Response,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    url_for,
+)
 from redis import Redis
 from ruamel import yaml
 
@@ -91,6 +100,16 @@ def setup_dalton_logging():
     RULECAT_SCRIPT = dalton_config.get("dalton", "rulecat_script")
     MAX_PCAP_FILES = dalton_config.getint("dalton", "max_pcap_files")
     ENABLE_QUEUE_CLEARING = dalton_config.getboolean("dalton", "enable_queue_clearing")
+    # Suricata rule syntax checking (optional; the linter container is a
+    # convenience, not a dependency - the controller fails open if it's
+    # unreachable). dalton.conf is the single source of truth: an absent or
+    # empty URL means the feature is off, which is what README.rst and
+    # dalton.conf tell operators removing the keys will do.
+    RULE_CHECK_URL = dalton_config.get("dalton", "rule_check_url", fallback="")
+    RULE_CHECK_TIMEOUT = dalton_config.getint(
+        "dalton", "rule_check_timeout", fallback=6
+    )
+    KEYWORDS_URL = dalton_config.get("dalton", "keywords_url", fallback="")
     DEBUG = dalton_config.getboolean("dalton", "debug")
     AUTH_PREFIX = dalton_config.get("dalton", "auth_prefix")
     AUTH_MAX = dalton_config.getint("dalton", "auth_max")
@@ -725,6 +744,148 @@ def get_engine_conf_file(sensor):
         return engine_config
 
 
+def _linter_fail_open(url, timeout, payload=None):
+    """GET a linter URL (or POST `payload` as JSON to it) and return the
+    parsed JSON object, or None on any failure. This backs an optional
+    convenience (rule syntax checking / keyword completion); a broken or
+    unreachable linter must never cost the page anything beyond that
+    feature, so every failure mode - missing config, connection refused,
+    timeout, non-200, unparseable body, and valid-JSON-but-not-an-object -
+    is treated identically as "unavailable".
+    """
+    if not url:
+        return None
+    try:
+        if payload is None:
+            req = urllib.request.Request(url)
+        else:
+            req = urllib.request.Request(
+                url,
+                data=json.dumps(payload).encode("utf-8"),
+                headers={"Content-Type": "application/json"},
+            )
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            if resp.status != 200:
+                return None
+            body = json.loads(resp.read())
+    except Exception:
+        return None
+    if not isinstance(body, dict):
+        return None
+    return body
+
+
+# The keyword payload only changes when the linter container is rebuilt, so
+# a successful fetch is served from memory for this long before the linter
+# is asked again. Page loads with the editor preference on would otherwise
+# each pull the full keyword list through the same two gunicorn workers that
+# serve syntax checks.
+KEYWORDS_CACHE_TTL = 600
+_keywords_cache = None
+_keywords_cache_time = 0.0
+
+
+@dalton_blueprint.route("/dalton/controller_api/rule_keywords", methods=["GET"])
+@check_user
+def api_get_rule_keywords():
+    """Proxy to the linter's /keywords endpoint (Suricata keyword completion
+    data and hover docs), cached in-process. A fetch younger than
+    KEYWORDS_CACHE_TTL is served from memory without asking the linter.
+    Fails open: any problem reaching the linter falls back to the last
+    successfully cached response (marked stale) so a linter outage degrades
+    completion to stale, not absent; only a never-yet-successful linter
+    reports unavailable. A module-level cache is enough for the
+    single-process dev server this controller runs on."""
+    global _keywords_cache, _keywords_cache_time
+    now = time.monotonic()
+    if _keywords_cache is not None and now - _keywords_cache_time < KEYWORDS_CACHE_TTL:
+        fresh = dict(_keywords_cache)
+        fresh["available"] = True
+        fresh["stale"] = False
+        return jsonify(fresh)
+    body = _linter_fail_open(KEYWORDS_URL, RULE_CHECK_TIMEOUT)
+    if body is not None:
+        _keywords_cache = dict(body)
+        _keywords_cache_time = now
+        body["available"] = True
+        body["stale"] = False
+        return jsonify(body)
+    if _keywords_cache is not None:
+        stale = dict(_keywords_cache)
+        stale["available"] = True
+        stale["stale"] = True
+        return jsonify(stale)
+    return jsonify({"available": False})
+
+
+# Must match MAX_BYTES in dalton-agent/linter/guard.py. Its own constant here
+# so the controller can refuse an oversized buffer before paying to parse it.
+MAX_RULE_CHECK_BYTES = 64 * 1024
+
+
+def _oversize_diagnostic():
+    """Shaped like a language-server diagnostic so the editor renders it inline
+    rather than treating an ordinary refusal as a failure."""
+    return {
+        "range": {
+            "start": {"line": 0, "character": 0},
+            "end": {"line": 0, "character": 0},
+        },
+        "message": (
+            f"Too much text to syntax check (limit {MAX_RULE_CHECK_BYTES // 1024} KB). "
+            "The rules can still be submitted as a job."
+        ),
+        "source": "Dalton",
+        "severity": 1,
+        "content": "",
+        "sid": 0,
+    }
+
+
+@dalton_blueprint.route("/dalton/controller_api/check_rules", methods=["POST"])
+@check_user
+def api_check_rules():
+    """Proxy to the linter's /check endpoint (Suricata rule syntax
+    checking). Fails open: any problem reaching the linter, or no linter
+    configured, reports {"available": false} rather than an HTTP error -
+    this backs an optional convenience and must never cost the page
+    anything beyond its own diagnostics. Job submission never goes near
+    this endpoint."""
+    # The linter's own limit is 64KB, but it only sees the buffer after two
+    # hops, and MAX_CONTENT_LENGTH here is a gigabyte because it is sized for
+    # pcap uploads. Without this a huge "rules" string is parsed, re-serialised
+    # and re-encoded in a development server that spawns an unbounded thread
+    # per request, before anything rejects it.
+    if request.content_length and request.content_length > MAX_RULE_CHECK_BYTES * 2:
+        return jsonify({"available": True, "diagnostics": [_oversize_diagnostic()]})
+
+    data = request.get_json(silent=True)
+    if not isinstance(data, dict):
+        data = {}
+    rules = data.get("rules") or ""
+    if not isinstance(rules, str):
+        return jsonify({"error": "'rules' must be a string"}), 400
+    if len(rules.encode("utf-8", "replace")) > MAX_RULE_CHECK_BYTES:
+        return jsonify({"available": True, "diagnostics": [_oversize_diagnostic()]})
+
+    payload = {
+        "rules": rules,
+        "engine_analysis": bool(data.get("engine_analysis", False)),
+    }
+    body = _linter_fail_open(RULE_CHECK_URL, RULE_CHECK_TIMEOUT, payload)
+    if body is None:
+        return jsonify({"available": False})
+    # Only the two fields the page uses, rather than whatever the linter
+    # returned, so a field added there later cannot leak through this proxy.
+    return jsonify(
+        {
+            "available": True,
+            "diagnostics": body.get("diagnostics", []),
+            "engine_version": body.get("engine_version"),
+        }
+    )
+
+
 @dalton_blueprint.route("/dalton/sensor_api/update/", methods=["POST"])
 # @auth_required('write')
 # status update from Dalton Agent

app/static/codemirror/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (C) 2017 by Marijn Haverbeke <marijn@haverbeke.berlin> and others
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

app/static/codemirror/addon/hint/show-hint.css

@@ -0,0 +1,37 @@
+.CodeMirror-hints {
+  position: absolute;
+  z-index: 10;
+  overflow: hidden;
+  list-style: none;
+
+  margin: 0;
+  padding: 2px;
+
+  -webkit-box-shadow: 2px 3px 5px rgba(0,0,0,.2);
+  -moz-box-shadow: 2px 3px 5px rgba(0,0,0,.2);
+  box-shadow: 2px 3px 5px rgba(0,0,0,.2);
+  border-radius: 3px;
+  border: 1px solid silver;
+
+  background: white;
+  font-size: 90%;
+  font-family: monospace;
+
+  max-height: 20em;
+  overflow-y: auto;
+  box-sizing: border-box;
+}
+
+.CodeMirror-hint {
+  margin: 0;
+  padding: 0 4px;
+  border-radius: 2px;
+  white-space: pre;
+  color: black;
+  cursor: pointer;
+}
+
+li.CodeMirror-hint-active {
+  background: #08f;
+  color: white;
+}