From b6644383d78417b2b2a83ffb759bc5c8f752e727 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Sat, 22 Aug 2026 20:00:34 +0800 Subject: [PATCH 1/4] =?UTF-8?q?test:=20=E6=96=B0=E5=A2=9E=20llm-compat=20?= =?UTF-8?q?=E6=8B=92=E7=BB=9D=E6=A3=80=E6=B5=8B=E8=B7=A8=E5=BA=93=E5=A5=91?= =?UTF-8?q?=E7=BA=A6=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在升级前锁定 7 格不变式轴表；长文含题材词与 on_all_refused 构造参数 在 v0.8.0 下预期失败，升级 v0.10.0 后应变绿。 Co-authored-by: Cursor --- tests/unit/test_refusal_contract.py | 162 ++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/unit/test_refusal_contract.py diff --git a/tests/unit/test_refusal_contract.py b/tests/unit/test_refusal_contract.py new file mode 100644 index 0000000..c113d03 --- /dev/null +++ b/tests/unit/test_refusal_contract.py @@ -0,0 +1,162 @@ +"""Cross-library refusal detection contract tests (llm-compat v0.10.0). + +Uses real llm_compat.refusal.detect_refusal — the library itself is never mocked. +Wire-format response dicts mirror what SyncLLMClient receives from providers. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +import pytest + +from llm_compat import ContentPolicyError +from llm_compat.refusal import detect_refusal + +from video_transcript_api.llm.core.errors import RetryableError, map_llm_compat_error + + +def _is_refusal(data: dict[str, Any]) -> bool: + """Normalize detect_refusal return across llm-compat API versions.""" + result = detect_refusal(data) + if isinstance(result, bool): + return result + return result.is_refusal + + +def _make_response(content: str | None, finish_reason: str = "stop") -> dict[str, Any]: + return { + "choices": [ + { + "message": {"content": content}, + "finish_reason": finish_reason, + } + ] + } + + +# >300 chars, legal/medical tone with repeated 违反 (production false-positive pattern) +LONG_LEGAL_SUMMARY = ( + "这是一段关于法律题材的长文总结，涵盖医疗合规与侵权责任。" + + "该行为可能违反相关法律法规，涉及敏感话题的讨论需要谨慎对待。" + * 15 +) + +# >300 chars, embed every legacy substring keyword from pre-v0.10.0 table +_LEGACY_KEYWORDS = ["无法提供", "涉及敏感", "不适合讨论", "违反"] +LONG_ALL_LEGACY_KEYWORDS = ( + "法律医疗题材总结。" + + "。".join( + f"段落{i}讨论{k}相关问题" + for i in range(30) + for k in _LEGACY_KEYWORDS + ) +) + + +REFUSAL_CONTRACT_CASES: list[tuple[str, str | None, str, bool, str]] = [ + ( + "long_with_wei_fan", + LONG_LEGAL_SUMMARY, + "stop", + False, + "long legal/medical summary containing wei_fan", + ), + ( + "long_all_legacy_keywords", + LONG_ALL_LEGACY_KEYWORDS, + "stop", + False, + "long text with all legacy keyword substrings", + ), + ( + "short_cn_refusal", + "抱歉，我无法提供这方面的帮助。", + "stop", + True, + "short CN first-person refusal at sentence start", + ), + ( + "short_en_refusal", + "I'm sorry, but I cannot assist with that request.", + "stop", + True, + "short EN first-person refusal", + ), + ( + "pseudo_refusal_with_turn", + "我不能协助这个请求，但可以换个角度说明。", + "stop", + False, + "pseudo refusal with concessive turn (but/however)", + ), + ( + "content_filter", + "ok", + "content_filter", + True, + "provider finish_reason content_filter", + ), + ( + "malformed_none_content", + None, + "stop", + True, + "None content with finish_reason stop", + ), +] + + +@pytest.mark.parametrize( + "case_id,content,finish_reason,expected,detail", + REFUSAL_CONTRACT_CASES, + ids=[c[0] for c in REFUSAL_CONTRACT_CASES], +) +def test_detect_refusal_contract( + case_id: str, + content: str | None, + finish_reason: str, + expected: bool, + detail: str, +) -> None: + """Table-driven contract: real detect_refusal on wire-format response dicts.""" + if case_id.startswith("long"): + text = content if isinstance(content, str) else "" + assert len(text) > 300, f"case {case_id} requires >300 chars, got {len(text)}" + + data = _make_response(content, finish_reason) + actual = _is_refusal(data) + assert actual == expected, f"case {case_id}: {detail}" + + +class TestSyncLLMClientRefusalPolicy: + """Construction-time on_all_refused must stay raise (not v0.10.0 default).""" + + @patch("video_transcript_api.llm.llm.SyncLLMClient") + def test_on_all_refused_raise_passed_at_construction(self, mock_client_cls) -> None: + from video_transcript_api.llm.llm import set_default_config + + config = { + "llm": { + "api_key": "test-key", + "base_url": "https://api.test.com/v1", + }, + } + set_default_config(config) + call_kwargs = mock_client_cls.call_args[1] + assert call_kwargs.get("on_all_refused") == "raise" + + +class TestContentPolicyErrorChain: + """All-refused terminal path: ContentPolicyError -> map_llm_compat_error -> RetryableError.""" + + def test_content_policy_error_maps_to_retryable(self) -> None: + err = ContentPolicyError( + "All models refused", + attempted_models=["deepseek-v4", "gemini-3-flash"], + raw_content="I cannot assist", + original_model="deepseek-v4", + ) + result = map_llm_compat_error(err) + assert isinstance(result, RetryableError) From cdf1ab0ffaa7deec7f3eef53fb08fc8fdc262a6e Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Sat, 22 Aug 2026 20:00:52 +0800 Subject: [PATCH 2/4] =?UTF-8?q?chore:=20=E5=8D=87=E7=BA=A7=20llm-compat=20?= =?UTF-8?q?pin=20=E8=87=B3=20v0.10.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复拒绝检测对长文题材词的无约束子串误判（上游 issue #22）。 Co-authored-by: Cursor --- pyproject.toml | 2 +- uv.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4ea76a0..b7ed24f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,4 +79,4 @@ exclude_lines = [ ] [tool.uv.sources] -llm-compat = { git = "https://github.com/zlxlabs/llm-compat.git", tag = "v0.8.0" } +llm-compat = { git = "https://github.com/zlxlabs/llm-compat.git", tag = "v0.10.0" } diff --git a/uv.lock b/uv.lock index 8b45488..16b6ce9 100644 --- a/uv.lock +++ b/uv.lock @@ -714,8 +714,8 @@ sdist = { url = "https://files.pythonhosted.org/packages/34/b8/aa7d6cf2d5efdd2fc [[package]] name = "llm-compat" -version = "0.8.0" -source = { git = "https://github.com/zlxlabs/llm-compat.git?tag=v0.8.0#bc57401ed98a6b52b458b8f2a5fed5ba70e9fd05" } +version = "0.10.0" +source = { git = "https://github.com/zlxlabs/llm-compat.git?tag=v0.10.0#7c6fa7fef00b4974e56fcb91e47ecf391600cca2" } dependencies = [ { name = "httpx", extra = ["socks"] }, { name = "pydantic" }, @@ -4494,7 +4494,7 @@ requires-dist = [ { name = "ffmpeg-python", specifier = ">=0.2.0" }, { name = "jinja2", specifier = ">=3.0.0" }, { name = "keyboard", specifier = "==0.13.5" }, - { name = "llm-compat", git = "https://github.com/zlxlabs/llm-compat.git?tag=v0.8.0" }, + { name = "llm-compat", git = "https://github.com/zlxlabs/llm-compat.git?tag=v0.10.0" }, { name = "loguru", specifier = "==0.7.0" }, { name = "markdown", specifier = ">=3.4.0" }, { name = "nh3", specifier = ">=0.2.0" }, From abe82fd4794aea147b864ffb294449fa1bc3e82f Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Sat, 22 Aug 2026 20:01:02 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20=E6=98=BE=E5=BC=8F=20on=5Fall=5Frefu?= =?UTF-8?q?sed=3Draise=20=E5=B9=B6=E7=A7=BB=E9=99=A4=E6=AD=BB=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 不使用 v0.10.0 默认 return_best，保持链耗尽时诚实标 failed； 删除未使用的 detect_provider import（v0.9.0 breaking）。 Co-authored-by: Cursor --- src/video_transcript_api/llm/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video_transcript_api/llm/llm.py b/src/video_transcript_api/llm/llm.py index d687333..374f6f3 100644 --- a/src/video_transcript_api/llm/llm.py +++ b/src/video_transcript_api/llm/llm.py @@ -19,7 +19,6 @@ from llm_compat.providers import ( build_request_payload, describe_from_payload, - detect_provider, ) # 注意：不能在模块顶层 `from .core.usage_context import ...`—— @@ -163,6 +162,7 @@ def set_default_config(config: Optional[Dict[str, Any]]) -> None: collector_api_key=llm_cfg.get("collector_api_key", ""), refusal_keywords_url=llm_cfg.get("refusal_keywords_url"), sensitive_detector=sensitive_detector, + on_all_refused="raise", ) if sensitive_detector: logger.info(f"[LLM] SyncLLMClient initialized with SensitiveDetector ({len(sensitive_detector._words)} words)") From 6f5d680605d0aad4a33ae6034e31f7a5f4e19f87 Mon Sep 17 00:00:00 2001 From: zj1123581321 Date: Sat, 22 Aug 2026 20:27:06 +0800 Subject: [PATCH 4/4] =?UTF-8?q?test:=20=E6=81=A2=E5=A4=8D=20set=5Fdefault?= =?UTF-8?q?=5Fconfig=20=E6=94=B9=E5=86=99=E7=9A=84=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E7=BA=A7=E5=85=A8=E5=B1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gate 主审 finding testing-global-client-state-leak： test_on_all_refused_raise_passed_at_construction 调真实 set_default_config， 把 mock 客户端写进 llm 模块的 _sync_client / _default_config 且不恢复。 已实证泄漏为真：临时关掉本 fixture 后，同会话内后置探针立刻断言失败 （global _sync_client leaked as MagicMock）。此前全量绿只是因为下游碰全局的 测试恰好都自己重设，把泄漏盖住了。 via [HAPI](https://hapi.run) Co-Authored-By: HAPI Agent-Executor: cc Agent-Session: 9dd95690-cd40-438a-aed2-eb0c1279caa7 Agent-Effort: xhigh --- tests/unit/test_refusal_contract.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unit/test_refusal_contract.py b/tests/unit/test_refusal_contract.py index c113d03..ed8cef4 100644 --- a/tests/unit/test_refusal_contract.py +++ b/tests/unit/test_refusal_contract.py @@ -133,6 +133,25 @@ def test_detect_refusal_contract( class TestSyncLLMClientRefusalPolicy: """Construction-time on_all_refused must stay raise (not v0.10.0 default).""" + @pytest.fixture(autouse=True) + def _restore_llm_module_globals(self): + """Restore module-level globals mutated by set_default_config(). + + set_default_config() writes _default_config and _sync_client. Without + this fixture the mock client built here leaks into any later test that + reads the module default instead of setting its own, making results + depend on collection order. + """ + from video_transcript_api.llm import llm as llm_mod + + saved_config = llm_mod._default_config + saved_client = llm_mod._sync_client + try: + yield + finally: + llm_mod._default_config = saved_config + llm_mod._sync_client = saved_client + @patch("video_transcript_api.llm.llm.SyncLLMClient") def test_on_all_refused_raise_passed_at_construction(self, mock_client_cls) -> None: from video_transcript_api.llm.llm import set_default_config