From c6c663e2ceb80ee8328d11a0e56934ed1f6bae6b Mon Sep 17 00:00:00 2001 From: Rafael Pupio Vieira <66076558+RafaelPupio@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:12:56 -0400 Subject: [PATCH 1/2] fix(models): cap local context for responsiveness --- .env.example | 5 +++++ docker-compose.gpu-amd.yml | 1 + docker-compose.gpu-nvidia.yml | 1 + docker-compose.yml | 1 + src/model_context.py | 19 +++++++++++++++++++ tests/test_model_context.py | 27 +++++++++++++++++++++++++++ 6 files changed, 54 insertions(+) diff --git a/.env.example b/.env.example index 61d874d55f..483f1caf24 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,11 @@ LLM_HOST=localhost # when started with OLLAMA_HOST=0.0.0.0:11434. # OLLAMA_BASE_URL=http://host.docker.internal:11434/v1 +# Cap context reported by local endpoints to keep long chats responsive. +# Remote/API endpoints are unaffected. Set to 0 or leave unset outside Compose +# to use the model's full reported context window. +# ODYSSEUS_LOCAL_CONTEXT_CAP=8192 + # Optional LM Studio URL. In Docker, host LM Studio is reachable here # when LM Studio is set to serve on all interfaces (0.0.0.0). # LM_STUDIO_URL=http://host.docker.internal:1234 diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 8d0cf16531..fa44d4dd50 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -69,6 +69,7 @@ services: - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} + - ODYSSEUS_LOCAL_CONTEXT_CAP=${ODYSSEUS_LOCAL_CONTEXT_CAP:-8192} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index 69331ffb67..b128447707 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -68,6 +68,7 @@ services: - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} + - ODYSSEUS_LOCAL_CONTEXT_CAP=${ODYSSEUS_LOCAL_CONTEXT_CAP:-8192} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.yml b/docker-compose.yml index 708e5df828..f337aa826b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,7 @@ services: - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} + - ODYSSEUS_LOCAL_CONTEXT_CAP=${ODYSSEUS_LOCAL_CONTEXT_CAP:-8192} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/src/model_context.py b/src/model_context.py index b6afa801e2..12f127e4e4 100644 --- a/src/model_context.py +++ b/src/model_context.py @@ -7,6 +7,7 @@ import ipaddress import logging +import os import sys from typing import Dict, List, Optional, Tuple @@ -252,6 +253,24 @@ def _get_context_length_cached(endpoint_url: str, model: str) -> Tuple[int, bool return _context_cache[cache_key] ctx, known = _query_context_length(endpoint_url, model) + # Large local context windows can make an otherwise healthy model appear + # hung: every turn must prefill the conversation before producing a token. + # Cloud/API endpoints keep their advertised window. + if is_local: + raw_cap = os.getenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "").strip() + try: + local_cap = int(raw_cap) if raw_cap else 0 + except ValueError: + logger.warning("Ignoring invalid ODYSSEUS_LOCAL_CONTEXT_CAP=%r", raw_cap) + local_cap = 0 + if local_cap > 0 and ctx > local_cap: + logger.info( + "Capping local context for %s from %s to %s tokens", + model, + ctx, + local_cap, + ) + ctx = local_cap # Only cache non-default values to allow retry on next request. # Local endpoints can restart with a different --max-model-len while keeping # the same model id, so always re-query them instead of serving stale cache. diff --git a/tests/test_model_context.py b/tests/test_model_context.py index 86872f30d4..dcc5fadbb5 100644 --- a/tests/test_model_context.py +++ b/tests/test_model_context.py @@ -206,6 +206,7 @@ def setup_method(self): model_context._catalog_ctx_cache.clear() def test_local_endpoint_requeries_same_model_after_restart(self, monkeypatch): + monkeypatch.delenv("ODYSSEUS_LOCAL_CONTEXT_CAP", raising=False) calls = [] def fake_query(endpoint_url, model): @@ -243,6 +244,32 @@ def fake_query(endpoint_url, model): assert second == 200000 assert len(calls) == 1 + def test_local_context_can_be_capped_for_interactive_latency(self, monkeypatch): + monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192") + monkeypatch.setattr( + model_context, "_query_context_length", lambda *_: (32768, True) + ) + + result = model_context.get_context_length_known( + "http://host.docker.internal:11434/v1/chat/completions", + "qwen3-coder:30b", + ) + + assert result == (8192, True) + + def test_local_context_cap_does_not_reduce_remote_models(self, monkeypatch): + monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192") + monkeypatch.setattr( + model_context, "_query_context_length", lambda *_: (200000, True) + ) + + result = model_context.get_context_length( + "https://api.openai.com/v1/chat/completions", + "gpt-5", + ) + + assert result == 200000 + def _proxy_db(self, monkeypatch): _install_endpoint_db(monkeypatch, [ types.SimpleNamespace( From a8be6a3511239bd68ca8d13ab0081725a339b1c5 Mon Sep 17 00:00:00 2001 From: RafaelPupio <66076558+RafaelPupio@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:21:01 -0400 Subject: [PATCH 2/2] test(models): cover local context cap boundaries --- tests/test_model_context.py | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_model_context.py b/tests/test_model_context.py index dcc5fadbb5..f4598d61bc 100644 --- a/tests/test_model_context.py +++ b/tests/test_model_context.py @@ -257,6 +257,51 @@ def test_local_context_can_be_capped_for_interactive_latency(self, monkeypatch): assert result == (8192, True) + @pytest.mark.parametrize( + "configured_cap", + [None, "", "0", "-1", "invalid", "65536"], + ) + def test_local_context_ignores_disabled_or_non_reducing_caps( + self, monkeypatch, configured_cap + ): + if configured_cap is None: + monkeypatch.delenv("ODYSSEUS_LOCAL_CONTEXT_CAP", raising=False) + else: + monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", configured_cap) + monkeypatch.setattr( + model_context, "_query_context_length", lambda *_: (32768, True) + ) + + result = model_context.get_context_length_known( + "http://host.docker.internal:11434/v1/chat/completions", + "qwen3-coder:30b", + ) + + assert result == (32768, True) + + def test_context_cap_does_not_reduce_configured_private_proxy( + self, monkeypatch + ): + monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192") + _install_endpoint_db(monkeypatch, [ + types.SimpleNamespace( + base_url="http://100.117.136.97:34521/v1", + endpoint_kind="proxy", + api_key="fake-key", + is_enabled=True, + ) + ]) + monkeypatch.setattr( + model_context, "_query_context_length", lambda *_: (32768, True) + ) + + result = model_context.get_context_length_known( + "http://100.117.136.97:34521/v1/chat/completions", + "proxied-model", + ) + + assert result == (32768, True) + def test_local_context_cap_does_not_reduce_remote_models(self, monkeypatch): monkeypatch.setenv("ODYSSEUS_LOCAL_CONTEXT_CAP", "8192") monkeypatch.setattr(