From 409bda6b5084349d3c4c88a29f40490ef3c468f9 Mon Sep 17 00:00:00 2001 From: proboscis Date: Mon, 10 Aug 2026 07:14:52 +0900 Subject: [PATCH 1/2] =?UTF-8?q?test:=20EffectBase.=5F=5Freduce=5Fex=5F=5F?= =?UTF-8?q?=20=E3=81=AE=20per-call=20copyreg=20=E8=A7=A3=E6=B1=BA=E3=82=92?= =?UTF-8?q?=E8=B5=A4=E3=81=A7=E5=9B=BA=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit free-threaded CPython 3.14 では import と module 属性参照が per-object 鍵を 取るため、`__reduce_ex__` が呼び出しのたびに copyreg を import していると 常駐 runtime で import 鍵が競合点になる。2026-08-07 の実測(ACP hypha 常駐 runtime): 948 threads が import 鍵で滞留し、機体全体で +19.5 GiB の swap 押し出し。native stack は _PyMutex_LockTimed→_PyParkingLot_Park→ __psynch_cvwait、到達元は PyImport_ImportModuleLevelObject→ import_ensure_initialized と _Py_module_getattro_impl、呼び手は doeff_vm.cpython-314t-darwin.so。 TDD 第 1 相(実装前・意図的に red): - tests/test_effect_base_reduce_ex_hot_path.py - 実行時の証明: warm-up 後に `__reduce_ex__` を 200 回叩き、 builtins.__import__ の呼び出しが 0 であること (実装前 = copyreg が 200 回 → red) - ソース上の証明: `__reduce_ex__` 本文に py.import が無く、 静的 PyOnceLock (COPYREG_NEWOBJ) 経由で解決していること - pickle 往復の既存挙動(reduce タプルの形・全 protocol・cloudpickle・ 入れ子・多スレッド同時実行)は不変であること - .semgrep.yaml: doeff-vm-no-per-call-copyreg-resolution を新設 (py.import("copyreg") / getattr("__newobj__") を doeff-vm 系 src で禁止) - tests/semgrep/fixtures/rust/…/python_generator_stream.rs: 改修前の形を bad fixture として常設し、rule 発火と shipped source での非発火を assert - docs/adr/enforcement-ledger.json: semgrep_rules 247→248 (ADR-DOE-ENFORCE-001 R5 の記帳) Co-Authored-By: Claude Opus 5 --- .semgrep.yaml | 23 +++ docs/adr/enforcement-ledger.json | 2 +- .../doeff-vm/src/python_generator_stream.rs | 9 ++ .../semgrep/test_vm_failfast_semgrep_rules.py | 28 ++++ tests/test_effect_base_reduce_ex_hot_path.py | 153 ++++++++++++++++++ 5 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 tests/semgrep/fixtures/rust/packages/doeff-vm/src/python_generator_stream.rs create mode 100644 tests/test_effect_base_reduce_ex_hot_path.py diff --git a/.semgrep.yaml b/.semgrep.yaml index 56238ed3..73d4094c 100644 --- a/.semgrep.yaml +++ b/.semgrep.yaml @@ -1401,6 +1401,29 @@ rules: include: - /packages/doeff-vm/src/pyvm.rs + - id: doeff-vm-no-per-call-copyreg-resolution + languages: [rust] + severity: ERROR + message: | + BANNED: resolving copyreg.__newobj__ per call (py.import("copyreg") / + getattr("__newobj__")). pickle hooks run on every effect instance, and + on free-threaded CPython 3.14 both import and module attribute lookup + take per-object locks — so a per-call resolution turns pickling into a + contention point. Measured 2026-08-07 in a long-lived multi-threaded + runtime: 948 threads parked on the import lock + (_PyMutex_LockTimed → _PyParkingLot_Park), reached from + PyImport_ImportModuleLevelObject → import_ensure_initialized and + _Py_module_getattro_impl, doeff_vm as the caller; 19.5 GiB paged out. + Resolve once into a static PyOnceLock — see COPYREG_NEWOBJ in + packages/doeff-vm/src/python_generator_stream.rs. + pattern-either: + - pattern-regex: py\.import\("copyreg"\) + - pattern-regex: getattr\("__newobj__"\) + paths: + include: + - "**/packages/doeff-vm/src/*.rs" + - "**/packages/doeff-vm-core/src/*.rs" + # tombstone(2026-07-14): 退役済みパターン no-is-instance-from-in-handlers の再発防止として意図的に維持。現行の非fixture対象ゼロは想定内。 - id: no-is-instance-from-in-handlers metadata: diff --git a/docs/adr/enforcement-ledger.json b/docs/adr/enforcement-ledger.json index f1f706b0..da307618 100644 --- a/docs/adr/enforcement-ledger.json +++ b/docs/adr/enforcement-ledger.json @@ -1,7 +1,7 @@ { "_comment": "ADR-DOE-ENFORCE-001 R5 anti-drop ratchet の台帳。enforcement 資産の数が黙って減る(または黙って増える)ことを tests/test_enforcement_ledger.py が禁止する。数を変える変更は、この台帳の明示的な更新を同じ変更セットに含めること。", "defadr_files": 22, - "semgrep_rules": 247, + "semgrep_rules": 248, "adr_deftest_enforcements": 30, "adr_defsemgrep_enforcements": 45, "adr_laws": 72 diff --git a/tests/semgrep/fixtures/rust/packages/doeff-vm/src/python_generator_stream.rs b/tests/semgrep/fixtures/rust/packages/doeff-vm/src/python_generator_stream.rs new file mode 100644 index 00000000..2fbdf5a3 --- /dev/null +++ b/tests/semgrep/fixtures/rust/packages/doeff-vm/src/python_generator_stream.rs @@ -0,0 +1,9 @@ +// Bad fixture for doeff-vm-no-per-call-copyreg-resolution: the pre-2026-08-10 +// shape of EffectBase.__reduce_ex__, which resolved copyreg.__newobj__ on +// every call and wedged free-threaded runtimes on the import lock. +pub fn reduce_ex_bad(slf: &Bound<'_, PyAny>) -> PyResult> { + let py = slf.py(); + let copyreg = py.import("copyreg")?; + let newobj = copyreg.getattr("__newobj__")?; + Ok(newobj.unbind()) +} diff --git a/tests/semgrep/test_vm_failfast_semgrep_rules.py b/tests/semgrep/test_vm_failfast_semgrep_rules.py index a4292c92..9978a66b 100644 --- a/tests/semgrep/test_vm_failfast_semgrep_rules.py +++ b/tests/semgrep/test_vm_failfast_semgrep_rules.py @@ -292,3 +292,31 @@ def test_koine_interactive_terminalize_rule_is_clean_on_fixed_policy() -> None: # M3 で消滅し、include が crate src のみの死に rule になるため。readiness # discard の不変量は現役側の同契約(sessionhost/launch.hy)の deftest 群が # 引き続き守る。rollback 座標 = git tag agentd-rust-final。 + + +def test_copyreg_per_call_resolution_rule_detects_pre_fix_reduce_ex() -> None: + """doeff-vm-no-per-call-copyreg-resolution は改修前の __reduce_ex__ 形に発火する。 + + 2026-08-07 の hypha 常駐 runtime 滞留(948 threads が import 鍵で park)の + 発生源だった「呼び出しごとの copyreg 解決」の回帰ガード。 + """ + fixture_root = REPO_ROOT / "tests/semgrep/fixtures/rust" + results = _semgrep_results( + REPO_ROOT / ".semgrep.yaml", + "packages/doeff-vm/src/python_generator_stream.rs", + cwd=fixture_root, + ) + + # py.import("copyreg") と getattr("__newobj__") の 2 行に発火する + assert _rule_start_lines(results, "doeff-vm-no-per-call-copyreg-resolution") == {6, 7} + + +def test_copyreg_per_call_resolution_rule_is_clean_on_shipped_source() -> None: + """出荷中の python_generator_stream.rs には発火しない(PyOnceLock 化済み)。""" + results = _semgrep_results( + REPO_ROOT / ".semgrep.yaml", + "packages/doeff-vm/src/python_generator_stream.rs", + cwd=REPO_ROOT, + ) + + assert _rule_start_lines(results, "doeff-vm-no-per-call-copyreg-resolution") == set() diff --git a/tests/test_effect_base_reduce_ex_hot_path.py b/tests/test_effect_base_reduce_ex_hot_path.py new file mode 100644 index 00000000..c9e32362 --- /dev/null +++ b/tests/test_effect_base_reduce_ex_hot_path.py @@ -0,0 +1,153 @@ +"""EffectBase.__reduce_ex__ の hot path 不変量 — copyreg 解決は 1 回だけ。 + +free-threaded CPython 3.14 では import と module 属性参照が per-object の鍵を +取る。`__reduce_ex__` が呼び出しのたびに `copyreg` を import していると、常駐 +runtime(多スレッド)では import 鍵が競合点になる。 + +実測(2026-08-07・ACP hypha 常駐 runtime): 948 threads が import 鍵で滞留し、 +機体全体で +19.5 GiB の swap 押し出し。滞留スレッドの native stack は +_PyMutex_LockTimed→_PyParkingLot_Park→__psynch_cvwait、到達元は +PyImport_ImportModuleLevelObject→import_ensure_initialized と +_Py_module_getattro_impl、呼び手は doeff_vm.cpython-314t-darwin.so。 + +不変量: `copyreg.__newobj__` の解決はプロセスで 1 回だけ(PyOnceLock)。 +hot path には import も module 属性参照も残さない。pickle の往復挙動は不変。 +""" + +import builtins +import copyreg +import pickle +import re +import threading +from pathlib import Path + +import cloudpickle +from doeff_vm import EffectBase + +ROOT = Path(__file__).resolve().parents[1] +STREAM_RS = ROOT / "packages" / "doeff-vm" / "src" / "python_generator_stream.rs" + + +class Ping(EffectBase): + def __init__(self, value, tag="t"): + self.value = value + self.tag = tag + + +def _reduce_ex_source() -> str: + """`__reduce_ex__` の本文だけを Rust ソースから切り出す。""" + src = STREAM_RS.read_text() + match = re.search(r"fn __reduce_ex__.*?\n \}", src, re.DOTALL) + assert match is not None, f"__reduce_ex__ が {STREAM_RS} に見つからない" + return match.group(0) + + +# --------------------------------------------------------------------------- +# 受入条件 (2): pickle 往復の既存挙動が変わらない +# --------------------------------------------------------------------------- + + +class TestEffectBasePickle: + def test_reduce_ex_shape(self): + """(copyreg.__newobj__, (cls,), __dict__) の 3 要素を返す。""" + eff = Ping(42) + reduced = eff.__reduce_ex__(2) + assert isinstance(reduced, tuple) + assert len(reduced) == 3 + assert reduced[0] is copyreg.__newobj__ + assert reduced[1] == (Ping,) + assert reduced[2] == {"value": 42, "tag": "t"} + + def test_pickle_roundtrip(self): + restored = pickle.loads(pickle.dumps(Ping(42))) + assert isinstance(restored, Ping) + assert restored.value == 42 + assert restored.tag == "t" + + def test_pickle_roundtrip_all_protocols(self): + for protocol in range(2, pickle.HIGHEST_PROTOCOL + 1): + restored = pickle.loads(pickle.dumps(Ping({"k": [1, 2]}), protocol)) + assert restored.value == {"k": [1, 2]}, f"protocol={protocol}" + + def test_cloudpickle_roundtrip(self): + restored = cloudpickle.loads(cloudpickle.dumps(Ping([1, 2, 3], tag="cp"))) + assert restored.value == [1, 2, 3] + assert restored.tag == "cp" + + def test_nested_effect(self): + restored = pickle.loads(pickle.dumps(Ping(Ping(7)))) + assert restored.value.value == 7 + + +# --------------------------------------------------------------------------- +# 受入条件 (1): hot path に import が無い(実行時の証明 + ソース上の証明) +# --------------------------------------------------------------------------- + + +class TestReduceExHotPathHasNoImport: + def test_no_import_machinery_per_call(self, monkeypatch): + """warm-up 後の `__reduce_ex__` は import 機構を一切叩かない。 + + `py.import()` は PyImport_Import 経由で builtins.__import__ を呼ぶため、 + `__import__` の呼び出し回数がそのまま import 鍵に触れた回数になる。 + (pickle.dumps 自体は save_global で __import__ を呼ぶので、ここでは + `__reduce_ex__` を直接叩いて hot path だけを測る。) + """ + eff = Ping(1) + eff.__reduce_ex__(2) # warm-up: 1 回だけの解決を済ませる + + seen: list[str] = [] + real_import = builtins.__import__ + + def counting_import(name, *args, **kwargs): + seen.append(name) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", counting_import) + for _ in range(200): + eff.__reduce_ex__(2) + + assert seen == [], f"__reduce_ex__ の hot path が import を呼んだ: {seen}" + + def test_concurrent_reduce_ex_is_consistent(self): + """多スレッドから同時に叩いても解決結果は同一かつ正しい。""" + results: list[object] = [] + errors: list[BaseException] = [] + barrier = threading.Barrier(8) + + def worker(): + try: + barrier.wait() + for _ in range(200): + results.append(Ping(1).__reduce_ex__(2)[0]) + except BaseException as exc: # スレッド内例外を回収して main で assert する + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [] + assert len(results) == 8 * 200 + assert all(r is copyreg.__newobj__ for r in results) + + def test_source_has_no_per_call_import(self): + """ソース上の証明: `__reduce_ex__` 本文に import / getattr 解決が無い。""" + body = _reduce_ex_source() + assert "py.import(" not in body, ( + "__reduce_ex__ の hot path に per-call import が復活している " + "(free-threaded 3.14 の import 鍵滞留の再発 — 2026-08-07 実測)" + ) + assert "COPYREG_NEWOBJ" in body, ( + "__newobj__ の解決は静的 PyOnceLock (COPYREG_NEWOBJ) 経由で 1 回だけ行うこと" + ) + + def test_source_caches_newobj_in_once_lock(self): + """静的 PyOnceLock に `copyreg.__newobj__` を保持していること。""" + src = STREAM_RS.read_text() + assert "PyOnceLock" in src, "PyOnceLock による 1 回きりの解決が無い" + assert re.search( + r"static\s+COPYREG_NEWOBJ\s*:\s*PyOnceLock>", src + ), "copyreg.__newobj__ を保持する静的 PyOnceLock が無い" From da187e324713184388129be4bd652c8bb852d634 Mon Sep 17 00:00:00 2001 From: proboscis Date: Mon, 10 Aug 2026 07:15:03 +0900 Subject: [PATCH 2/2] =?UTF-8?q?fix(doeff-vm):=20copyreg.=5F=5Fnewobj=5F=5F?= =?UTF-8?q?=20=E3=82=92=20PyOnceLock=20=E3=81=A7=201=20=E5=9B=9E=E3=81=A0?= =?UTF-8?q?=E3=81=91=E8=A7=A3=E6=B1=BA=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EffectBase.__reduce_ex__ の hot path から import と module 属性参照を外す。 静的 `COPYREG_NEWOBJ: PyOnceLock>` に解決済み callable を保持し、 2 回目以降は refcount 加算だけで済ませる(free-threaded 3.14 の import 鍵・ module getattr 鍵に触れない)。返す reduce タプルの形は不変 — (copyreg.__newobj__, (cls,), __dict__)。 前相の red がすべて green: - __reduce_ex__ 200 回で builtins.__import__ 呼び出し 0(改修前 = 200) - pickle 往復(全 protocol・cloudpickle・入れ子)は不変 - 8 スレッド × 200 回同時実行でも解決結果は copyreg.__newobj__ で一致 - doeff-vm/doeff-vm-core の Rust src から py.import は 0 箇所になった ADR-DOE-ENFORCE-001 の TDD+semgrep 手順に従い、旧形は doeff-vm-no-per-call-copyreg-resolution が恒久ガードする。 Co-Authored-By: Claude Opus 5 --- .../doeff-vm/src/python_generator_stream.rs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/doeff-vm/src/python_generator_stream.rs b/packages/doeff-vm/src/python_generator_stream.rs index 6cecf081..02d99425 100644 --- a/packages/doeff-vm/src/python_generator_stream.rs +++ b/packages/doeff-vm/src/python_generator_stream.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::PyStopIteration; use pyo3::prelude::*; use pyo3::pyclass::{PyTraverseError, PyVisit}; +use pyo3::sync::PyOnceLock; use pyo3::types::PyString; use doeff_vm_core::do_ctrl::DoCtrl; @@ -34,6 +35,20 @@ use doeff_vm_core::value::Value; #[derive(Debug)] pub struct PyEffectBase; +/// `copyreg.__newobj__`, resolved once per process. +/// +/// Free-threaded CPython 3.14 takes a per-object lock both for `import` and +/// for module attribute lookup, so resolving this on every `__reduce_ex__` +/// call makes pickling a contention point instead of a refcount bump. +/// Measured on 2026-08-07 in a long-lived multi-threaded runtime embedding +/// this extension: 948 threads parked on the import lock +/// (_PyMutex_LockTimed → _PyParkingLot_Park → __psynch_cvwait), reached from +/// PyImport_ImportModuleLevelObject → import_ensure_initialized and from +/// _Py_module_getattro_impl, with doeff_vm as the caller; the machine paged +/// out 19.5 GiB. The cell holds the resolved callable so the hot path never +/// touches the import machinery again. +static COPYREG_NEWOBJ: PyOnceLock> = PyOnceLock::new(); + #[pymethods] impl PyEffectBase { #[new] @@ -48,10 +63,14 @@ impl PyEffectBase { /// Pickle support: return (copyreg.__newobj__, (cls,), __dict__). /// copyreg.__newobj__(cls) calls cls.__new__(cls), then pickle sets /// obj.__dict__.update(state) for the third element. + /// + /// `copyreg.__newobj__` comes from COPYREG_NEWOBJ — resolved once per + /// process. Do NOT import or read a module attribute here: this runs on + /// every pickle of every effect, and on free-threaded builds those take + /// locks (see the COPYREG_NEWOBJ doc comment for the measured wedge). fn __reduce_ex__(slf: &Bound<'_, Self>, _protocol: i32) -> PyResult> { let py = slf.py(); - let copyreg = py.import("copyreg")?; - let newobj = copyreg.getattr("__newobj__")?; + let newobj = COPYREG_NEWOBJ.import(py, "copyreg", "__newobj__")?; let cls = slf.get_type(); let args = pyo3::types::PyTuple::new(py, &[cls.as_any()])?; let state = slf.getattr("__dict__")?;