fix(models): cap local context for responsive chats

.env.example

@@ -16,6 +16,11 @@ LLM_HOST=localhost
 # when started with OLLAMA_HOST=0.0.0.0:11434.
 # OLLAMA_BASE_URL=http://host.docker.internal:11434/v1
 
+# Cap context reported by local endpoints to keep long chats responsive.
+# Remote/API endpoints are unaffected. Set to 0 or leave unset outside Compose
+# to use the model's full reported context window.
+# ODYSSEUS_LOCAL_CONTEXT_CAP=8192
+
 # Optional LM Studio URL. In Docker, host LM Studio is reachable here
 # when LM Studio is set to serve on all interfaces (0.0.0.0).
 # LM_STUDIO_URL=http://host.docker.internal:1234

docker-compose.gpu-amd.yml

@@ -69,6 +69,7 @@ services:
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      - ODYSSEUS_LOCAL_CONTEXT_CAP=${ODYSSEUS_LOCAL_CONTEXT_CAP:-8192}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.gpu-nvidia.yml

@@ -68,6 +68,7 @@ services:
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      - ODYSSEUS_LOCAL_CONTEXT_CAP=${ODYSSEUS_LOCAL_CONTEXT_CAP:-8192}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.yml

@@ -57,6 +57,7 @@ services:
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      - ODYSSEUS_LOCAL_CONTEXT_CAP=${ODYSSEUS_LOCAL_CONTEXT_CAP:-8192}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

src/model_context.py

@@ -7,6 +7,7 @@
 
 import ipaddress
 import logging
+import os
 import sys
 from typing import Dict, List, Optional, Tuple
 
@@ -252,6 +253,24 @@ def _get_context_length_cached(endpoint_url: str, model: str) -> Tuple[int, bool
         return _context_cache[cache_key]
 
     ctx, known = _query_context_length(endpoint_url, model)
+    # Large local context windows can make an otherwise healthy model appear
+    # hung: every turn must prefill the conversation before producing a token.
+    # Cloud/API endpoints keep their advertised window.
+    if is_local:
+        raw_cap = os.getenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "").strip()
+        try:
+            local_cap = int(raw_cap) if raw_cap else 0
+        except ValueError:
+            logger.warning("Ignoring invalid ODYSSEUS_LOCAL_CONTEXT_CAP=%r", raw_cap)
+            local_cap = 0
+        if local_cap > 0 and ctx > local_cap:
+            logger.info(
+                "Capping local context for %s from %s to %s tokens",
+                model,
+                ctx,
+                local_cap,
+            )
+            ctx = local_cap
     # Only cache non-default values to allow retry on next request.
     # Local endpoints can restart with a different --max-model-len while keeping
     # the same model id, so always re-query them instead of serving stale cache.

tests/test_model_context.py

@@ -206,6 +206,7 @@ def setup_method(self):
         model_context._catalog_ctx_cache.clear()
 
     def test_local_endpoint_requeries_same_model_after_restart(self, monkeypatch):
+        monkeypatch.delenv("ODYSSEUS_LOCAL_CONTEXT_CAP", raising=False)
         calls = []
 
         def fake_query(endpoint_url, model):
@@ -243,6 +244,77 @@ def fake_query(endpoint_url, model):
         assert second == 200000
         assert len(calls) == 1
 
+    def test_local_context_can_be_capped_for_interactive_latency(self, monkeypatch):
+        monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192")
+        monkeypatch.setattr(
+            model_context, "_query_context_length", lambda *_: (32768, True)
+        )
+
+        result = model_context.get_context_length_known(
+            "http://host.docker.internal:11434/v1/chat/completions",
+            "qwen3-coder:30b",
+        )
+
+        assert result == (8192, True)
+
+    @pytest.mark.parametrize(
+        "configured_cap",
+        [None, "", "0", "-1", "invalid", "65536"],
+    )
+    def test_local_context_ignores_disabled_or_non_reducing_caps(
+        self, monkeypatch, configured_cap
+    ):
+        if configured_cap is None:
+            monkeypatch.delenv("ODYSSEUS_LOCAL_CONTEXT_CAP", raising=False)
+        else:
+            monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", configured_cap)
+        monkeypatch.setattr(
+            model_context, "_query_context_length", lambda *_: (32768, True)
+        )
+
+        result = model_context.get_context_length_known(
+            "http://host.docker.internal:11434/v1/chat/completions",
+            "qwen3-coder:30b",
+        )
+
+        assert result == (32768, True)
+
+    def test_context_cap_does_not_reduce_configured_private_proxy(
+        self, monkeypatch
+    ):
+        monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192")
+        _install_endpoint_db(monkeypatch, [
+            types.SimpleNamespace(
+                base_url="http://100.117.136.97:34521/v1",
+                endpoint_kind="proxy",
+                api_key="fake-key",
+                is_enabled=True,
+            )
+        ])
+        monkeypatch.setattr(
+            model_context, "_query_context_length", lambda *_: (32768, True)
+        )
+
+        result = model_context.get_context_length_known(
+            "http://100.117.136.97:34521/v1/chat/completions",
+            "proxied-model",
+        )
+
+        assert result == (32768, True)
+
+    def test_local_context_cap_does_not_reduce_remote_models(self, monkeypatch):
+        monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192")
+        monkeypatch.setattr(
+            model_context, "_query_context_length", lambda *_: (200000, True)
+        )
+
+        result = model_context.get_context_length(
+            "https://api.openai.com/v1/chat/completions",
+            "gpt-5",
+        )
+
+        assert result == 200000
+
     def _proxy_db(self, monkeypatch):
         _install_endpoint_db(monkeypatch, [
             types.SimpleNamespace(