diff --git a/com.avaloq.tools.ddk.xtext.test/META-INF/MANIFEST.MF b/com.avaloq.tools.ddk.xtext.test/META-INF/MANIFEST.MF index 47dbd33084..89c2348819 100644 --- a/com.avaloq.tools.ddk.xtext.test/META-INF/MANIFEST.MF +++ b/com.avaloq.tools.ddk.xtext.test/META-INF/MANIFEST.MF @@ -32,7 +32,8 @@ Require-Bundle: com.avaloq.tools.ddk.xtext, junit-platform-suite-engine, org.eclipse.xtext.testing, org.opentest4j -Import-Package: com.avaloq.tools.ddk.check.runtime.test.core, +Import-Package: com.avaloq.tools.ddk.caching, + com.avaloq.tools.ddk.check.runtime.test.core, com.avaloq.tools.ddk.check.test.core, com.avaloq.tools.ddk.check.test.runtime.tests, com.avaloq.tools.ddk.check.ui.test, diff --git a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/XtextTestSuite.java b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/XtextTestSuite.java index b3d62132d0..a048096a49 100644 --- a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/XtextTestSuite.java +++ b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/XtextTestSuite.java @@ -14,9 +14,12 @@ import org.junit.platform.suite.api.Suite; import com.avaloq.tools.ddk.xtext.builder.XtextBuildTriggerTest; +import com.avaloq.tools.ddk.xtext.builder.resourceloader.ParallelResourceLoaderTest; +import com.avaloq.tools.ddk.xtext.formatting.ExtendedFormattingConfigBasedStreamSortLocatorsTest; import com.avaloq.tools.ddk.xtext.jupiter.formatter.FormatterTest; import com.avaloq.tools.ddk.xtext.linking.AbstractFragmentProviderTest; import com.avaloq.tools.ddk.xtext.linking.ShortFragmentProviderTest; +import com.avaloq.tools.ddk.xtext.naming.QualifiedNameLookupFormalTest; import com.avaloq.tools.ddk.xtext.naming.QualifiedNamePatternTest; import com.avaloq.tools.ddk.xtext.naming.QualifiedNameSegmentTreeLookupTest; import com.avaloq.tools.ddk.xtext.resource.AbstractSelectorFragmentProviderTest; @@ -37,12 +40,15 @@ AbstractSelectorFragmentProviderTest.class, ResourceDescriptionDeltaTest.class, XtextBuildTriggerTest.class, + ParallelResourceLoaderTest.class, FormatterTest.class, QualifiedNamePatternTest.class, BugAig1084.class, RuntimeProjectUtilTest.class, QualifiedNameSegmentTreeLookupTest.class, - PatternAwareEObjectDescriptionLookUpTest.class}) + QualifiedNameLookupFormalTest.class, + PatternAwareEObjectDescriptionLookUpTest.class, + ExtendedFormattingConfigBasedStreamSortLocatorsTest.class}) // @Format-On public class XtextTestSuite { } diff --git a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoaderTest.java b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoaderTest.java new file mode 100644 index 0000000000..38a7e385a5 --- /dev/null +++ b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoaderTest.java @@ -0,0 +1,94 @@ +/******************************************************************************* + * Copyright (c) 2026 Avaloq Group AG and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Avaloq Group AG - initial API and implementation + *******************************************************************************/ +package com.avaloq.tools.ddk.xtext.builder.resourceloader; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.eclipse.emf.common.util.URI; +import org.eclipse.emf.ecore.resource.Resource; +import org.eclipse.emf.ecore.resource.ResourceSet; +import org.eclipse.emf.ecore.resource.impl.ResourceImpl; +import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl; +import org.eclipse.xtext.builder.resourceloader.IResourceLoader.LoadOperation; +import org.eclipse.xtext.builder.resourceloader.IResourceLoader.LoadOperationException; +import org.eclipse.xtext.builder.resourceloader.IResourceLoader.Sorter; +import org.eclipse.xtext.resource.IResourceServiceProvider; +import org.eclipse.xtext.resource.persistence.SourceLevelURIsAdapter; +import org.eclipse.xtext.ui.resource.IResourceSetProvider; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.inject.Guice; + + +/** + * Tests for {@link ParallelResourceLoader}. + */ +@SuppressWarnings("nls") +public class ParallelResourceLoaderTest { + + private static final URI SLOW_URI = URI.createURI("platform:/resource/project/slow.test"); + private static final long TIMEOUT_MILLIS = 50; + + @Disabled("Documents LDR-1, see formal/BUGS.md; enable with the fix") + @Test + public void timeoutKeepsPendingResultAvailable() { + CountDownLatch loadReleased = new CountDownLatch(1); + Resource slowResource = new ResourceImpl(SLOW_URI); + ParallelResourceLoader loader = new ParallelResourceLoader(resourceSetProvider(), new Sorter.NoSorting(), 1, -1) { + @Override + protected Resource loadResource(final URI uri, final ResourceSet localResourceSet, final ResourceSet parentResourceSet) { + Uninterruptibles.awaitUninterruptibly(loadReleased); + return slowResource; + } + }; + IResourceServiceProvider.Registry registry = mock(IResourceServiceProvider.Registry.class); + Guice.createInjector(binder -> binder.bind(IResourceServiceProvider.Registry.class).toInstance(registry)).injectMembers(loader); + loader.setTimeout(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + + ResourceSet parent = new ResourceSetImpl(); + SourceLevelURIsAdapter.setSourceLevelUris(parent, Collections.emptySet()); + LoadOperation operation = loader.create(parent, null); + try { + operation.load(List.of(SLOW_URI)); + + LoadOperationException timeout = assertThrows(LoadOperationException.class, operation::next); + assertInstanceOf(TimeoutException.class, timeout.getCause()); + assertTrue(operation.hasNext(), "a timed-out poll must not consume the pending result"); + + loadReleased.countDown(); + assertSame(slowResource, operation.next().getResource()); + } finally { + loadReleased.countDown(); + operation.cancel(); + } + } + + private static IResourceSetProvider resourceSetProvider() { + IResourceSetProvider provider = mock(IResourceSetProvider.class); + when(provider.get(any())).thenAnswer(invocation -> new ResourceSetImpl()); + return provider; + } + +} diff --git a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/formatting/ExtendedFormattingConfigBasedStreamSortLocatorsTest.java b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/formatting/ExtendedFormattingConfigBasedStreamSortLocatorsTest.java new file mode 100644 index 0000000000..a7ea86ba3e --- /dev/null +++ b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/formatting/ExtendedFormattingConfigBasedStreamSortLocatorsTest.java @@ -0,0 +1,96 @@ +/******************************************************************************* + * Copyright (c) 2026 Avaloq Group AG and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Avaloq Group AG - initial API and implementation + *******************************************************************************/ +package com.avaloq.tools.ddk.xtext.formatting; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.xtext.XtextFactory; +import org.eclipse.xtext.formatting.impl.AbstractFormattingConfig.ElementLocator; +import org.eclipse.xtext.formatting.impl.FormattingConfig; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import com.avaloq.tools.ddk.xtext.formatting.locators.FixedLocator; +import com.avaloq.tools.ddk.xtext.formatting.locators.NoFormatLocator; + + +/** + * Tests the private {@code sortLocators} of {@link ExtendedFormattingConfigBasedStream}, which must order column locators (opening before closing, then by + * column) regardless of other locators interleaved in the list. + */ +@SuppressWarnings({"PMD.JUnitAssertionsShouldIncludeMessage", "nls"}) +public class ExtendedFormattingConfigBasedStreamSortLocatorsTest { + + private static final String OPEN_1 = "open@1"; + private static final String OPEN_5 = "open@5"; + + private final FormattingConfig config = new FormattingConfig(null, null, null); + + private final ExtendedFormattingConfigBasedStream stream = new ExtendedFormattingConfigBasedStream(null, null, config, null, null, false, null); + + @Test + public void testFixedLocatorsAreSortedByColumn() throws ReflectiveOperationException { + List locators = new ArrayList<>(List.of(opening(5), opening(1))); + sortLocators(locators); + assertEquals(List.of(OPEN_1, OPEN_5), describeFixedLocators(locators)); + } + + @Disabled("Documents RO-1, see formal/BUGS.md; enable with the fix") + @Test + public void testFixedLocatorsAreSortedByColumnAcrossOtherLocator() throws ReflectiveOperationException { + List locators = new ArrayList<>(List.of(opening(5), new NoFormatLocator(config), opening(1))); + sortLocators(locators); + assertEquals(List.of(OPEN_1, OPEN_5), describeFixedLocators(locators)); + } + + @Disabled("Documents RO-1, see formal/BUGS.md; enable with the fix") + @Test + public void testOpeningFixedLocatorPrecedesClosingAcrossOtherLocator() throws ReflectiveOperationException { + List locators = new ArrayList<>(List.of(closing(5), new NoFormatLocator(config), opening(5))); + sortLocators(locators); + assertEquals(List.of(OPEN_5, "close@5"), describeFixedLocators(locators)); + } + + private FixedLocator opening(final int column) { + return new FixedLocator(config, column, false, false, false); + } + + private FixedLocator closing(final int column) { + FixedLocator locator = new FixedLocator(config, column, false, false, false); + locator.after(XtextFactory.eINSTANCE.createKeyword()); + return locator; + } + + private void sortLocators(final List locators) throws ReflectiveOperationException { + Method method = ExtendedFormattingConfigBasedStream.class.getDeclaredMethod("sortLocators", List.class); + method.setAccessible(true); // NOPMD AvoidAccessibilityAlteration - sortLocators is private + try { + method.invoke(stream, locators); + } catch (InvocationTargetException e) { + throw new IllegalStateException(e.getCause()); + } + } + + private static List describeFixedLocators(final List locators) { + List result = new ArrayList<>(); + for (ElementLocator locator : locators) { + if (locator instanceof FixedLocator fixed) { + result.add((fixed.getLeft() == null ? "open@" : "close@") + fixed.getColumn()); + } + } + return result; + } +} diff --git a/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameLookupFormalTest.java b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameLookupFormalTest.java new file mode 100644 index 0000000000..1de806dae5 --- /dev/null +++ b/com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameLookupFormalTest.java @@ -0,0 +1,186 @@ +/******************************************************************************* + * Copyright (c) 2026 Avaloq Group AG and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Avaloq Group AG - initial API and implementation + *******************************************************************************/ +package com.avaloq.tools.ddk.xtext.naming; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.eclipse.emf.ecore.EClass; +import org.eclipse.emf.ecore.EcoreFactory; +import org.eclipse.emf.ecore.EcorePackage; +import org.eclipse.xtext.naming.QualifiedName; +import org.eclipse.xtext.resource.EObjectDescription; +import org.eclipse.xtext.resource.IEObjectDescription; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import com.avaloq.tools.ddk.xtext.resource.PatternAwareEObjectDescriptionLookUp; + + +/** + * Failing-first regression tests for the counterexamples found by the Lean model in {@code formal/trie/lean} (see NOTES.md there, findings B1-B10). Each + * test asserts the intended behaviour: a pattern lookup returns exactly the values whose names the pattern matches, as the reference + * {@link TreeSetLookup} is meant to. + */ +@SuppressWarnings({"nls", "PMD.JUnitAssertionsShouldIncludeMessage"}) +// CHECKSTYLE:CONSTANTS-OFF +public class QualifiedNameLookupFormalTest { + + private static final String MAX = String.valueOf(Character.MAX_VALUE); + + // B1: upperExclusive() of a single-segment "*" / "**" is "!", which excludes every name at or above '!'. + @Disabled("Documents TRIE-2, see formal/BUGS.md; enable with the fix") + @Test + public void testTopLevelWildcardFindsSingleSegmentNames() { + assertEquals(List.of("b"), sorted(tree(false, name("b"), "b").get(pattern("*"), false))); + assertEquals(List.of("b"), sorted(treeSet(name("b"), "b").get(pattern("*"), false))); + } + + @Disabled("Documents TRIE-2, see formal/BUGS.md; enable with the fix") + @Test + public void testTopLevelRecursiveWildcardFindsAllNames() { + assertEquals(List.of("b", "bc"), sorted(tree(false, name("b"), "b", name("b", "c"), "bc").get(pattern("**"), false))); + } + + // B2: '!' is not the successor of a segment; "a " (and any suffix below '!') falls inside [a, a!). + @Disabled("Documents TRIE-3, see formal/BUGS.md; enable with the fix") + @Test + public void testExactPatternExcludesNameWithLowCharSuffix() { + assertFalse(pattern("a").matches(name("a "))); + assertEquals(List.of(), sorted(tree(false, name("a "), "a_").get(pattern("a"), false))); + assertEquals(List.of(), sorted(treeSet(name("a "), "a_").get(pattern("a"), false))); + } + + @Disabled("Documents TRIE-3, see formal/BUGS.md; enable with the fix") + @Test + public void testChildWildcardExcludesSiblingWithLowCharSuffix() { + assertEquals(List.of("ab"), sorted(tree(false, name("a ", "x"), "a_x", name("a", "b"), "ab").get(pattern("a", "*"), false))); + assertEquals(List.of(), sorted(treeSet(name("a ", "x"), "a_x").get(pattern("a", "*"), false))); + } + + // B3: a wildcard-free pattern matches() longer names, while both lookups return only the exact name. + @Disabled("Documents TRIE-4, see formal/BUGS.md; enable with the fix") + @Test + public void testExactPatternMatchesOnlyEqualLength() { + assertEquals(List.of("a"), sorted(tree(false, name("a"), "a", name("a", "b"), "ab").get(pattern("a"), false))); + assertFalse(pattern("a").matches(name("a", "b"))); + } + + // B4: QualifiedNamePattern.compareTo treats an empty pattern segment as smaller than anything (TreeSetLookup only). + @Disabled("Documents TRIE-10, see formal/BUGS.md; enable with the fix") + @Test + public void testTreeSetLookupEmptySegmentPattern() { + assertFalse(pattern("", " ").matches(name("", ""))); + assertEquals(List.of(), sorted(treeSet(name("", ""), "e").get(pattern("", " "), false))); + } + + // B5: put() increments size even when the value is already mapped. + @Disabled("Documents TRIE-5, see formal/BUGS.md; enable with the fix") + @Test + public void testSizeCountsMappingsOnce() { + QualifiedNameSegmentTreeLookup lookup = tree(false, name("a"), "v", name("a"), "v"); + assertEquals(1L, lookup.getStatistics().getEntries()); + lookup.remove(name("a"), "v"); + assertEquals(0L, lookup.getStatistics().getEntries()); + } + + // B6: getMappings() drops blank intermediate segments. + @Disabled("Documents TRIE-7, see formal/BUGS.md; enable with the fix") + @Test + public void testGetMappingsKeepsBlankSegments() { + assertEquals(List.of(name(" ", "a")), new ArrayList<>(tree(false, name(" ", "a"), "v").getMappings("v"))); + assertEquals(List.of(name("", "a")), new ArrayList<>(tree(false, name("", "a"), "v").getMappings("v"))); + } + + // B7: with shareValues=true, a child sharing its parent's value array is reported once for excludeDuplicates=false. + @Disabled("Documents TRIE-8, see formal/BUGS.md; enable with the fix") + @Test + public void testValueSharingKeepsMultiplicity() { + assertEquals(List.of("v", "v"), sorted(tree(true, name("a"), "v", name("a", "b"), "v").get(pattern("a**"), false))); + } + + // B8: PatternAwareEObjectDescriptionLookUp filters case-sensitive pattern queries with name.matches(name), which is always false. + @Disabled("Documents TRIE-1, see formal/BUGS.md; enable with the fix") + @Test + public void testCaseSensitivePatternQuery() { + PatternAwareEObjectDescriptionLookUp lookUp = new PatternAwareEObjectDescriptionLookUp(List.of(description("Foo"), description("FooBar"), description("foo"))); + List result = new ArrayList<>(); + lookUp.getExportedObjects(EcorePackage.Literals.ECLASS, pattern("Foo*"), false).forEach(d -> result.add(d.getName().toString())); + assertEquals(List.of("Foo", "FooBar"), sorted(result)); + } + + // B9: (char) (c + 1) wraps for U+FFFF, and a stored "￿" coincides with the tree's sentinel node. + @Disabled("Documents TRIE-9, see formal/BUGS.md; enable with the fix") + @Test + public void testMaxCharPattern() { + assertDoesNotThrow(() -> new TreeSetLookup().get(pattern(MAX + "*"), false)); + assertEquals(List.of("m"), sorted(tree(false, name(MAX), "m").get(pattern(MAX), false))); + } + + // B10: glob patterns (createFromGlobs) miss matches: "!" upper bound, non-recursive walk of "*"-suffixed globs, case-insensitive regexps. + @Disabled("Documents TRIE-12, TRIE-13, TRIE-14, see formal/BUGS.md; enable with the fix") + @Test + public void testGlobLookupsFindMatches() { + assertEquals(List.of("b"), sorted(tree(false, name("b"), "b").get(QualifiedNamePattern.createFromGlobs("*"), false))); + assertEquals(List.of("ax"), sorted(tree(false, name("a", "x"), "ax").get(QualifiedNamePattern.createFromGlobs("a*"), false))); + // whatever the case semantics of globs, a lookup must return what matches() accepts + QualifiedNamePattern upperCaseGlob = QualifiedNamePattern.createFromGlobs("F*"); + List expected = upperCaseGlob.matches(name("foo")) ? List.of("foo") : List.of(); + assertEquals(expected, sorted(tree(false, name("foo"), "foo").get(upperCaseGlob, false))); + } + + @Disabled("Documents TRIE-15, see formal/BUGS.md; enable with the fix") + @Test + public void testGlobWithEmptyLastSegment() { + assertDoesNotThrow(() -> QualifiedNamePattern.createFromGlobs("a", "").matches(name("a", ""))); + } + + private static QualifiedName name(final String... segments) { + return QualifiedName.create(segments); + } + + private static QualifiedNamePattern pattern(final String... segments) { + return QualifiedNamePattern.create(segments); + } + + private static QualifiedNameSegmentTreeLookup tree(final boolean shareValues, final Object... namesAndValues) { + QualifiedNameSegmentTreeLookup lookup = new QualifiedNameSegmentTreeLookup<>(String.class, shareValues); + for (int i = 0; i < namesAndValues.length; i += 2) { + lookup.put((QualifiedName) namesAndValues[i], (String) namesAndValues[i + 1]); + } + return lookup; + } + + private static TreeSetLookup treeSet(final Object... namesAndValues) { + TreeSetLookup lookup = new TreeSetLookup<>(); + for (int i = 0; i < namesAndValues.length; i += 2) { + lookup.put((QualifiedName) namesAndValues[i], (String) namesAndValues[i + 1]); + } + return lookup; + } + + private static List sorted(final Collection values) { + List result = new ArrayList<>(values); + result.sort(null); + return result; + } + + private static IEObjectDescription description(final String name) { + EClass eClass = EcoreFactory.eINSTANCE.createEClass(); + eClass.setName(name); + return EObjectDescription.create(QualifiedName.create(name), eClass); + } +} diff --git a/formal/.gitignore b/formal/.gitignore new file mode 100644 index 0000000000..dea2a48c1e --- /dev/null +++ b/formal/.gitignore @@ -0,0 +1,27 @@ +# Tools fetched locally and harness output. +.tools/ +.check/ + +# Lean build output. +**/.lake/ + +# TLC state and metadirs. +**/states/ +**/meta/ +**/meta-*/ + +# Logs and generated run artefacts of the model-checking scripts. +*.log +*.out +**/tla/runs/ +**/tla/out/ +**/tla/logs/ + +# Compiled Java of the cross-checks. +**/java-check/out/ +**/java-check/out-fixed/ +readonly/comparator/out/ + +# Machine-specific classpaths and third-party source extracts +**/classpath*.txt +find-refs/.src/ diff --git a/formal/BUGS.md b/formal/BUGS.md new file mode 100644 index 0000000000..b676fa7237 --- /dev/null +++ b/formal/BUGS.md @@ -0,0 +1,1070 @@ +# Formal-verification campaign: defect catalogue and fix plans + +This file catalogues the defects found by the formal-verification spike in DDK, with a verification record and a fix plan for each. It accompanies a reference draft PR that is not meant to be merged as a whole: the fixes are meant to land as the small, self-contained PRs in [section 5](#5-proposed-fix-pr-sequence). + +## How to reproduce + +- **Models:** `formal/check.sh` runs every TLC matrix against its golden verdicts, builds the Lean projects, checks axioms and runs the orphan-test check. `--full` adds the slow TLC runs; `--only=` (`parallel-loader`, `binary-storage`, `find-refs`, `trie`, `pipeline`, `orphans`) restricts it. Per-model details are in each `formal//{tla,lean}/NOTES.md`; the campaign log is `formal/REPORT.md`. +- **Failing-first Java tests:** this PR registers them in `XtextTestSuite`/`XtextUiTestSuite`, with every method that fails on master annotated `@Disabled("Documents …")` so CI stays green. To see one fail, delete its `@Disabled` and run the aggregator: `mvn verify -f ./ddk-parent/pom.xml -pl :com.avaloq.tools.ddk.xtext.test` (on Linux, wrap the command in `xvfb-run`). Do not rely on Tycho `-Dtest=…` to select a single test, because it can silently match nothing. + +## 1. Method and legend + +1. **Blind modelling.** Each component was modelled independently in TLA+ (TLC) and Lean 4 (kernel-checked proofs plus bounded search), without being told about suspected bugs. The release pipeline was modelled the same way. Read-only code passes (`ro-*` agents) covered the formatter comparator, test wiring and a trie self-match. +2. **Counterexample → finding.** Each violation was replayed against the Java source and turned into a finding with a trace and `file:line` locations. +3. **Skeptics.** Three independent skeptics tried to refute each finding or narrow it. **Phase A** verified the TRIE, PIPE and RO findings per originating agent, before cataloguing. **Phase B** verified LDR-2/3, REF-1/3–9, STO-1–8 and TRIE-16 after cataloguing, and recorded per skeptic whether the claim was overstated and a suggested severity. **Phase C** (a later sweep) put LDR-1 and REF-2, already confirmed by a failing Java test, and the 9 observations in [4.3](#43-verified-observations) through the same three skeptics, recording the same fields. +4. **Failing-first Java test**, where the interleaving could be made deterministic. +5. **Fix plans.** REF and STO each got two competing plans (A and B) and a judge. LDR, TRIE, PIPE and RO each got a single plan, so there is no judge record. + +Every finding records **Verification** (votes) and **Origin** (the agent, its finding ID and its own verdict). + +**Editorial rules** + +1. **Status.** CONFIRMED = a failing-first Java test reproduces it, or at least 2 of 3 skeptics upheld it. PLAUSIBLE = 1 of 3 upheld. REFUTED = 0 of 3 upheld. A finding whose originating verdict was PLAUSIBLE but which 3/3 skeptics upheld is CONFIRMED and marked "(promoted: originating verdict PLAUSIBLE, 3/3 upheld)": TRIE-11, TRIE-16, PIPE-6, PIPE-7, RO-6. A finding raised as an observation with no verdict takes its status from the votes alone (REF-7, REF-8, REF-9, STO-8). +2. **Severity** is the user-visible impact reachable in DDK or in its typical downstream consumers. **high** = wrong results, or an aborted, hung or blocked build or release, on a path DDK or a typical consumer reaches. **medium** = a reachable race or inconsistency whose impact is recoverable, or a high-class impact behind a narrow or downstream-only trigger. **low** = narrow trigger, cosmetic, latent (no production caller), or test/CI hygiene with no shipped-code impact. When skeptics called a claim overstated, the finding records the narrowed claim, and the severity is lowered if the narrowing reduces reachability. Binary-model storage is disabled on DDK's own target platforms (`DefaultXtextTargetPlatform:81` and `NullXtextTargetPlatform:74` return a null `IBinaryModelStore`, so `MCBS:1547` leaves storage off), so STO impact is downstream-only and STO severities are capped at medium. A finding confirmed by a failing Java test on master is never rated below a finding with the same impact and no test. +3. **Observations.** Adjacent issues raised by judges, planners or skeptics outside the original findings get a stable ID (`-A`) and are listed in [4.3](#43-verified-observations). They went through phase C and carry a status and severity under rules 1 and 2, but are counted separately from the 51 catalogued findings. Fix plans reference them as "optional". + +**Abbreviations** + +| Short | Path | +|---|---| +| `PRL` | `com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoader.java` | +| `MCBS` | `com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java` | +| `FRSRCP` | `com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProvider.java` | +| `QNP` | `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNamePattern.java` | +| `QNSTL` | `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameSegmentTreeLookup.java` | +| `PAEDL` | `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/resource/PatternAwareEObjectDescriptionLookUp.java` | +| `EFCBS` | `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/formatting/ExtendedFormattingConfigBasedStream.java` | +| `RSF` / `DLRSF` | Xtext `ResourceStorageFacade` / DDK `DirectLinkingResourceStorageFacade` | +| `QNLFT` | `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameLookupFormalTest.java` | +| `PAEDLT` | `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/resource/PatternAwareEObjectDescriptionLookUpTest.java` | +| `SLT` | `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/formatting/ExtendedFormattingConfigBasedStreamSortLocatorsTest.java` | +| `PRLT` | `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoaderTest.java` | +| `FRSRCPT` | `com.avaloq.tools.ddk.xtext.ui.test/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProviderTest.java` | + +## 2. Summary + +Test column: **disabled: X** = a test in this PR that fails on master, carried `@Disabled` (see [How to reproduce](#how-to-reproduce)). **script** = `formal/readonly/orphans/check-test-reachability.sh`. **proposed** = no test yet; the fix plan names the test to add. **(javac)** = red-on-master and green-with-fix were shown with the javac harness in `formal/trie/lean/java-check`; confirmation under the Tycho aggregator is still pending. LDR-1 and REF-2 were shown red under the aggregator. + +| ID | Sev | Status | Title | Test | Fix PR | +|---|---|---|---|---|---| +| LDR-1 | medium | CONFIRMED | Poll timeout decrements `toProcess`; build aborts as cancelled | disabled: `PRLT` (documents the first fix; [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) replaced it with its own tests) | [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) (draft) | +| LDR-2 | low | CONFIRMED | Interrupted builder livelocks the cluster loop (buffered queues) | proposed | LDR-PR2 | +| LDR-3 | low | CONFIRMED | Worker thread leaks in `put()` after `cancel()` if a load swallows the interrupt | proposed | LDR-PR3 | +| REF-1 | medium | CONFIRMED | UI deadlock: Reset handler `syncExec`s while holding the listeners monitor | proposed | REF-PR5 | +| REF-2 | low | CONFIRMED | `ConcurrentModificationException` in `inputChanged` during a running search | `FRSRCPT`, enabled on master by [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552) | [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552) (merged) | +| REF-3 | medium | CONFIRMED | Lost update: UIUpdater clears the scheduled flag after its `isEmpty()` check | proposed | REF-PR2 | +| REF-4 | low | CONFIRMED | `resourceNode` check-then-act creates duplicate roots, loses references | proposed | REF-PR4 | +| REF-5 | medium | CONFIRMED | Old search's node leaks into new view (clear before `removeListener`) | proposed | REF-PR3 | +| REF-6 | low | CONFIRMED | Same reference shown twice | proposed | REF-PR4 | +| REF-7 | low | CONFIRMED | Tree node `children` read/written concurrently | proposed | REF-PR6 | +| REF-8 | low | CONFIRMED | `descriptionsChanged` mutates tree concurrently with search thread | proposed | REF-PR6 | +| REF-9 | low | CONFIRMED | Exception in `runInUIThread` leaves the scheduled flag stuck | proposed | REF-PR2 | +| STO-1 | medium | CONFIRMED | URI removed from sources before its binary is written (MCBS:656) | proposed | STO-PR1 | +| STO-2 | low | CONFIRMED | Data race on the sources `HashSet` | proposed | STO-PR2 | +| STO-3 | low | CONFIRMED | Loaders read partial or stale binaries of dependencies | proposed | STO-PR1 | +| STO-4 | medium | CONFIRMED | Stores dropped by `shutdownNow` leave stale binaries across builds | proposed | STO-PR3 | +| STO-5 | low | CONFIRMED | Running stores not awaited after `shutdownNow` | proposed | STO-PR3 | +| STO-6 | low | PLAUSIBLE | Detached resource still stored after a link exception | proposed | STO-PR4 | +| STO-7 | — | REFUTED | Swallowed `IOException` keeps old binary while URI leaves sources | none | no fix | +| STO-8 | low | CONFIRMED | `InterruptedException` in the storage await swallowed | proposed | STO-PR5 | +| TRIE-1 | high | CONFIRMED | Case-sensitive pattern queries always empty (pattern matched against itself) | `PAEDLT`, enabled on master by [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551); disabled: `QNLFT` | [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551) (merged) | +| TRIE-2 | high | CONFIRMED | Top-level `*`/`**` upper bound `"!"` misses almost every name | disabled: `QNLFT` | TRIE-PR2 | +| TRIE-3 | low | CONFIRMED | `'!'` is not a segment successor; low-char names leak into results | disabled: `QNLFT` | TRIE-PR3 | +| TRIE-4 | low | CONFIRMED | Wildcard-free `matches()` accepts longer names | disabled: `QNLFT` | TRIE-PR4 | +| TRIE-5 | low | CONFIRMED | Statistics size over-counts duplicate put/putAll | disabled: `QNLFT` | TRIE-PR6 | +| TRIE-6 | low | CONFIRMED | `putAll` stores duplicates; one remove leaves value mapped | proposed | TRIE-PR6 | +| TRIE-7 | low | CONFIRMED | `getMappings` drops blank intermediate segments | disabled: `QNLFT` | TRIE-PR7 | +| TRIE-8 | low | CONFIRMED | `shareValues=true` collapses multiplicities | disabled: `QNLFT` | TRIE-PR8 | +| TRIE-9 | low | CONFIRMED | U+FFFF wraps the upper bound, collides with sentinel | disabled: `QNLFT` (stays red with `fix-plan.patch`) | TRIE-PR10 (issue first) | +| TRIE-10 | low | CONFIRMED | `TreeSetLookup` spurious results for empty-segment patterns | disabled: `QNLFT` | TRIE-PR5 | +| TRIE-11 | low | CONFIRMED (promoted) | `initializeFrom` aliases the source's mutable tree | proposed (after contract decision) | TRIE-PR11 (issue first) | +| TRIE-12 | low | CONFIRMED | Globs inherit `"!"` bound; wildcard-free glob matches nothing | disabled: `QNLFT` (part a only); rest proposed | TRIE-PR2, TRIE-PR3, TRIE-PR9 | +| TRIE-13 | low | CONFIRMED | Globs ending in `*` walk only one level | disabled: `QNLFT` | TRIE-PR9 | +| TRIE-14 | low | CONFIRMED | Glob regexps case-insensitive, bounds case-sensitive | disabled: `QNLFT` | TRIE-PR9 | +| TRIE-15 | low | CONFIRMED | Glob `matches()` SIOOBE on empty last segment | disabled: `QNLFT` | TRIE-PR9 | +| TRIE-16 | low | CONFIRMED (promoted) | `put(QualifiedName.EMPTY, v)` increments size then throws | proposed | TRIE-PR6 | +| PIPE-1 | high | CONFIRMED | Maintenance builds always fail the Tycho baseline gate | proposed | PIPE-PR4 | +| PIPE-2 | high | CONFIRMED | Next release version from global highest tag | proposed | [#1550](https://github.com/dsldevkit/dsl-devkit/pull/1550) (merged) | +| PIPE-3 | high | CONFIRMED | Tag pushed before release repo exists; can be orphaned | proposed | PIPE-PR6 | +| PIPE-4 | medium | CONFIRMED | `p2/snapshots/latest` can point at a non-master build | proposed | PIPE-PR3 | +| PIPE-5 | medium | CONFIRMED | Shared concurrency group cancels pending runs across refs | proposed | PIPE-PR7 | +| PIPE-6 | low | CONFIRMED (promoted) | Publish re-run cannot succeed after `gh release create` | proposed | PIPE-PR5 | +| PIPE-7 | low | CONFIRMED (promoted) | 8-char short SHA may differ between clones | proposed | PIPE-PR2 | +| RO-1 | medium | CONFIRMED | `sortLocators` comparator not a total order | disabled: `SLT` | RO-PR1 | +| RO-2 | low | CONFIRMED | `com.avaloq.tools.ddk.test.ui.test` tests never run | script (exits 1) | RO-PR3 | +| RO-3 | low | CONFIRMED | Comparator makes `Collections.sort` throw for ≥32 locators | proposed (scratch driver `MinSize.java` only) | RO-PR1 | +| RO-4 | low | CONFIRMED | `ErrorLogListenerTest` missing from its own bundle suite | script (exits 1) | RO-PR3 | +| RO-5 | low | CONFIRMED | `ErrorLogListenerTest` does not test ignoring; stale location | proposed | RO-PR3 | +| RO-6 | low | CONFIRMED (promoted) | `DeChKeyboardLayoutTest` stale layout name, leaks SWTBot prefs | proposed | RO-PR3 | +| RO-7 | low | CONFIRMED | CI guard cannot detect orphaned tests | script (proposed replacement guard) | RO-PR5 | +| RO-8 | low | CONFIRMED | Aggregator selects empty placeholder `CheckCfgUiTestSuite` | script (INFO) | RO-PR4 | + +## 3. Findings and fix plans by cluster + +### 3.1 LDR — `ParallelResourceLoader` + +#### LDR-1 — Poll timeout decrements `toProcess`, so the build aborts as cancelled without anyone cancelling + +- **Severity / status:** medium / CONFIRMED. **Verification:** failing Java test on master, green with `formal/f1-fix.patch`; Lean proves P1 for all sizes once fixed; phase C 3/3 upheld, 3 overstated; suggested severity medium, low, low. **Origin:** TLA+ round 1 F1 (CONFIRMED), Lean round 1 F1 (CONFIRMED). +- **Fix PR:** [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) (draft) — decrement only on a delivered result; a timeout abandons the whole load operation, and `writeResources` then cancels the build instead of leaving the remaining resources unindexed. +- **Locations:** PRL:220, PRL:225, MCBS:531, MCBS:603. +- **Claim:** `ParallelLoadOperation.next()` runs `toProcess--` right after `poll()`, even when `poll()` timed out and returned null (PRL:220-221). `hasNext()` is then false while a URI is still queued, and MCBS:531-536 logs `NO_MORE_RESOURCES`, calls `cancel()` and throws `OperationCanceledException`. All queue kinds are affected. One resource taking more than `MAX_WAIT_TIME` (300 s) to load is enough. Upstream Xtext has the same code. A second consumer, `writeResources` (MCBS:1077; code reading only, not modelled), ends its loop early on the same drift and silently drops the timed-out URI from indexing. +- **Narrowed (skeptics):** One slow resource is not enough: the builder's poll must wait the full 300 s with no worker delivering, in practice because the last outstanding load is stuck. The drift shows at once only if the timed-out load was the last one outstanding; otherwise it shows at the cluster end. The rollback then forces a full build. `writeResources` ends one iteration early and silently skips indexing one resource. The plain one-line fix would turn a hung load into an unbounded re-poll, hence the bounded-retry fix in [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553). **Severity rationale:** lowered from high to medium: reachable in DDK with its default bindings, but the trigger is rare and a naive fix trades the spurious cancel for a liveness problem. +- **Trace:** queue={u1}, toProcess=1 → `next()` poll times out, toProcess→0, `LoadOperationException(null, TimeoutException)`, u1 stays queued (MCBS:603) → loop head: queue non-empty but `!hasNext()` → `cancel()` + OCE. +- **Models:** `formal/parallel-loader/tla/NOTES.md`, `formal/parallel-loader/lean/NOTES.md`, `formal/f1-fix.patch`, `formal/REPORT.md`. +- **Test:** `PRLT#timeoutKeepsPendingResultAvailable`, `@Disabled("Documents LDR-1…")`. Fails on master at :76 (`hasNext()` false after the timeout). It documents the first variant of the fix, which kept waiting; [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) abandons the operation instead and brings its own tests. + +#### LDR-2 — Interrupted builder thread livelocks the cluster loop with `LinkedBlockingQueue`/`ArrayBlockingQueue` + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 2 overstated; suggested severity low ×3. **Origin:** TLA+ round 1 F2 (CONFIRMED, external trigger), Lean round 1 F2 (CONFIRMED; kernel-checked `livelock_trace_kernel`). +- **Locations:** PRL:222, MCBS:1359, MCBS:585. +- **Claim:** With the interrupt flag set, `poll()` on `LinkedBlockingQueue`/`ArrayBlockingQueue` throws `InterruptedException` even when a result is waiting. PRL:222-230 restores the flag and throws a fake timeout. `pollForCancellation` sleeps uninterruptibly (MCBS:1359-1365), so the flag survives, and the loop spins every ~5 s with a bogus 300 s warning until the user cancels. `SynchronousQueue` (queueSize 0, the default) does not livelock. An external interrupt is required. +- **Narrowed (skeptics):** DDK's own bindings (`ClusteringModule`) always use queueSize 0, and nothing in the Eclipse build path is known to interrupt the builder thread. Only a downstream `getParallelLoader(n, bufferSize != 0)` plus a leaked interrupt reaches the livelock. With `SynchronousQueue` a leftover interrupt still causes 5 s stalls and fake timeout warnings. **Severity rationale:** lowered from medium because the livelock is unreachable with DDK's bindings. +- **Trace:** worker puts result → builder interrupted → `poll` throws although an item is present → fake timeout → uninterruptible sleep, no cancel → `hasNext()` true → repeat. +- **Models:** `formal/parallel-loader/tla/NOTES.md`, `formal/parallel-loader/lean/NOTES.md`. +- **Test:** proposed (LDR-PR2): interrupted `pollForCancellation` throws OCE quickly, via a test subclass seam. + +#### LDR-3 — Executor thread leaks in `put()` after `cancel()` when a load swallows the interrupt + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 1 F3 (CONFIRMED, conditional). Not in Lean. +- **Locations:** PRL:367, PRL:289. +- **Claim:** `cancel()` relies only on the `shutdownNow()` interrupt to unblock `publishLoadResult`'s `put()` (PRL:367). If `doLoadResource` cleared the flag, a worker finishing after cancel blocks forever in `put()` on a `SynchronousQueue` or a full `ArrayBlockingQueue`. +- **Narrowed (skeptics):** No DDK/Xtext/EMF load path is shown to clear the flag. Each occurrence leaks up to `nThreads` (≤4) idle non-daemon threads and their retained resources; the build does not hang. Triggers include the finally-block `cancel()` after a 300 s timeout (MCBS:680) and the cluster-end cancel, not only a user cancel. Hardening. +- **Trace:** worker loading u1 → `shutdownNow` interrupts it → load completes having cleared the flag → `put()` on a dead `SynchronousQueue` blocks forever (WorkersQuiesce violated). +- **Models:** `formal/parallel-loader/tla/NOTES.md`. +- **Test:** proposed (LDR-PR3): `cancelDoesNotLeakWorkerWhenLoadSwallowsInterrupt`. + +#### LDR fix plans + +Single plan, no judge. The three findings are independent: Lean `fixTimeout_alone_still_livelocks` and `fixInterrupt_alone_still_aborts` show LDR-1 and LDR-2 each need their own fix. + +**LDR-PR1 (LDR-1)** + +- **Opened as [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) (draft).** Its first version gave up after 3 consecutive timeouts, which amounts to a longer timeout and still drops one result. It now decrements only on a delivered result, and a timeout abandons the whole load operation; `writeResources` then cancels the build, as the linking loop already did. This avoids both the miscount and the unbounded re-poll of the plain one-line fix (see Risks). +- **Fix:** In `ParallelLoadOperation.next()` (PRL:219-224), decrement only when `poll()` returned a result: `if (result != null) { toProcess--; }` (exactly `formal/f1-fix.patch`). Keep `@SuppressFBWarnings("AT_NONATOMIC_OPERATIONS_ON_SHARED_VARIABLE")`; only the builder thread touches the counter. MCBS needs no change. The same line fixes the `writeResources` consumer; say so in the PR body. +- **Files:** PRL; `com.avaloq.tools.ddk.xtext.builder/META-INF/MANIFEST.MF`; `com.avaloq.tools.ddk.xtext.builder/pom.xml`; `PRLT` (enable); `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/XtextTestSuite.java`. +- **Version bumps:** `com.avaloq.tools.ddk.xtext.builder` 17.3.1 → 17.3.2 (MANIFEST `.qualifier`, pom `-SNAPSHOT`), done only by the first of LDR-PR1/2/3 and STO-PR1..5 to land in the cycle. `xtext.test`: none (already 17.3.2). +- **Test strategy:** `timeoutKeepsPendingResultAvailable` overrides `loadResource(URI, ResourceSet, ResourceSet)` to block on a latch, sets a 50 ms timeout, and asserts `next()` throws `LoadOperationException(TimeoutException)` while `hasNext()` stays true; after the latch is released, `next()` returns the slow resource. Before the PR, parameterise over queueSize {-1, 0, 1}: TLA+ shows the bug for all three, and the spike test covers only -1. No thread is interrupted. +- **Risks:** A truly hung load now logs a timeout about every 305 s instead of ending as a spurious cancel, so it looks like a build that never finishes. User cancel still works. A cap on consecutive timeouts, if wanted, is a follow-up. No API change. +- **Model evidence:** TLA+ `ParallelLoader.tla` violates Bookkeeping, AbortOnlyOnCancel and BuildCompletes (3-step trace); `ParallelLoaderFixed` passes full_s/m/l/xl for all queue kinds; `variants/AblateFix1.tla` violates Bookkeeping again; the planted variant is caught. Lean `Proof.lean` `P1_fixTimeout` (inductive, all configs, no `sorry`); `Results.lean` `original_timeout_violates_P1/P2(_all_queues)`, `timeout_trace_kernel`. +- **Upstream:** Shared: Xtext 2.42 `ParallelResourceLoader.java:229-230` has the same unconditional decrement, mostly masked because upstream NPEs on a null result and its `ClusteringBuilderState` has no `hasNext()` guard. Worth one low-priority Xtext issue together with LDR-3. + +**LDR-PR2 (LDR-2)** + +- **Fix:** At the top of `pollForCancellation(IProgressMonitor)` (MCBS:1359): `if (Thread.interrupted()) { LOGGER.warn(...); throw new OperationCanceledException(); }`. It is called only from the three load-failure catch blocks (MCBS:587, 1103, 1122), so one site covers the cluster loop and `writeResources`. The OCE is thrown from inside the catch, so `catch(Exception)` does not swallow it. The flag is consumed on purpose: leaving it set would short-circuit `awaitBinaryStorageExecutorTermination` in the finally (MCBS:684). Widen `pollForCancellation` from private to protected as a test seam. Rejected: throwing OCE from PRL `next()` (swallowed by MCBS:585 `catch(Exception)`), and a loop-head `isInterrupted()` check (Lean's fixInterrupt), which misses the `writeResources` loop. +- **Files:** MCBS; builder MANIFEST/pom (if not yet bumped); `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderStateTest.java` (new); `XtextTestSuite.java`. +- **Version bumps:** builder 17.3.1 → 17.3.2 only if no earlier LDR/STO PR bumped it. `xtext.test`: none. +- **Test strategy:** An anonymous subclass of `MonitoredClusteringBuilderState(mock(IXtextTargetPlatformManager))` exposes `pollForCancellation` (a subclass, because package-private access across OSGi bundles fails). Interrupt the current thread; assert OCE well under the 5 s polling timeout and that the flag is clear. On master the call sleeps 5 s and returns. A finally block calls `Thread.interrupted()`. Optional precondition test in `PRLT` (queueSize -1 and 1): an interrupted `next()` fails fast with a fake timeout. A build-level test needs a test language with bufferSize ≠ 0 and is out of scope. +- **Risks:** Any interrupt that coincides with a failed `next()` now cancels the build, including with `SynchronousQueue`. `pollForCancellation` becomes protected API in an exported package; the baseline comparator is content-only, so a micro bump suffices. Must land before STO-PR5 (STO-8), whose interrupt restore at MCBS:865 would otherwise make the livelock reachable from an internal source. +- **Model evidence:** TLA+ `live_int_arr1`/`live_int_unb` violate BuilderStops/BuildCompletes; `live_int_sync` passes; `AblateFix2` brings the violation back for buffered queues only. Lean `livelock_trace_kernel`, `original_interrupt_violates_P3`, `original_interrupt_sync_ok`, `fixBoth_ok` (P1-P3, all 4 queue kinds). **Caveat:** neither model uses this exact placement; see [open question 3](#6-open-questions). `writeResources` is code reading only. +- **Upstream:** Partly shared: Xtext 2.42 (`ParallelResourceLoader.java:231-236`) also turns an interrupt into a missing result, but the livelock (uninterruptible poll plus `hasNext()` guard) is DDK-specific. At most mention it in the LDR-1/LDR-3 issue. + +**LDR-PR3 (LDR-3)** + +- **Fix:** Add `private volatile boolean cancelled` to `ParallelLoadOperation`, set first in `cancel()` (PRL:289). In `publishLoadResult` (PRL:365-371), replace `put()` with a timed `offer(result, PUBLISH_RETRY_MILLIS≈100, MILLISECONDS)` loop that exits when published or cancelled and restores the interrupt on `InterruptedException`. Give the loop a non-empty body (PMD `EmptyControlStatement`). Handoff semantics unchanged; no API change (`publishLoadResult` is private). +- **Files:** PRL; builder MANIFEST/pom (if not yet bumped); `PRLT` (add test). +- **Version bumps:** as LDR-PR2. +- **Test strategy:** `cancelDoesNotLeakWorkerWhenLoadSwallowsInterrupt`, parameterised over queueSize 0 and 1, with -1 as a control that also passes on master. The `loadResource` override records the worker, counts down `started`, awaits `release` uninterruptibly, swallows the interrupt with `Thread.interrupted()`, and returns a resource. The test awaits `started`, calls `cancel()`, releases, joins the worker with a 5 s bound and asserts it is dead. A finally block releases and interrupts the worker. +- **Risks:** Low. Blocked workers wake about every 100 ms. A load that never returns is not helped. Frame the PR as hardening. +- **Model evidence:** TLA+ `live_swallow_sync` and `leak_arr1` violate WorkersQuiesce; FIX-3 in `ParallelLoaderFixed.tla` passes full_s/m/l/xl; `AblateFix3` violates it on full_s_sync and full_m_arr1. Lean does not model swallowed interrupts. +- **Upstream:** Shared: Xtext 2.42 `ParallelResourceLoader.java:215-219` publishes with an unbounded `put()`. Same Xtext issue as LDR-1. + +### 3.2 REF — find references (`FastReferenceSearchResultContentProvider`) + +The class is not bound anywhere in DDK. Every REF finding affects only downstream products that bind it. REF-4, REF-7 and REF-8 are DDK-only: upstream Xtext mutates the tree only on the UI thread. + +#### REF-1 — UI deadlock: Reset handler `syncExec`s while holding the listeners monitor + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase B 3/3 upheld, 2 overstated; suggested severity medium ×3. **Origin:** TLA+ round 2 refs F2 (CONFIRMED by code reading), Lean round 2 refs F2 (CONFIRMED, timing window); javap of Xtext `ReferenceSearchResult` shows `fireEvent` and `removeListener` both lock `listeners`. +- **Locations:** FRSRCP:178, FRSRCP:113. +- **Claim:** `ReferenceSearchResult.fireEvent` calls listeners while holding the listeners monitor. The Reset handler calls `Display.syncExec` (FRSRCP:178). If the UI thread is switching to another search at that moment, it blocks on the same monitor, and both threads wait forever: the workbench freezes. The Xtext superclass has no `syncExec`. +- **Narrowed (skeptics):** The UI thread actually blocks first in Xtext `ReferenceSearchViewPage.setInput:147` (`removeListener(labelUpdater)`), before FRSRCP:113. The window exists only at job start and needs rapid user input (a new Find References, a history pick, or removing the query). **Severity rationale:** a hard hang, but lowered from high because the window is narrow and the class is downstream-only. +- **Trace:** job start: `ReferenceQuery.run` → `reset()` → `fireEvent(Reset)` takes `listeners` → provider `syncExec` waits for UI. Meanwhile UI: `setInput(B)` → `A.removeListener` blocks on `listeners` → deadlock. +- **Models:** `formal/find-refs/tla/NOTES.md`, `formal/find-refs/lean/NOTES.md`, `formal/find-refs/tla/traces/orig_NoDeadlock.txt`. +- **Test:** proposed (REF-PR5): park the UI thread and run a real `ReferenceSearchResult.reset()` on another thread. + +#### REF-2 — `ConcurrentModificationException` in `inputChanged` while the search is still running + +- **Severity / status:** low / CONFIRMED. **Verification:** failing Java test on master (CME), green with `fix-cme.patch`; phase C 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs F6 (CONFIRMED, racy), Lean round 2 refs F4 (CONFIRMED). +- **Fix PR:** [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552) (merged) — iterate a `Lists.newArrayList` copy and skip nulls. +- **Locations:** FRSRCP:118. +- **Claim:** `inputChanged` iterates the live `getMatchingReferences()` list while the search thread's `accept()` appends to it without a lock. The CME escapes `viewer.setInput` and leaves the page half-initialised: the listener is registered but the input is not set. The Xtext superclass has the same live iteration. An unsynchronised copy is not enough on its own: in JDK 9+ `ArrayList.clear()` sets size=0 before nulling the slots, but a copy can still see null slots when `toArray` reads the old size and then copies during `clear()`, or when a concurrent grow pads with nulls (REF judge correction). +- **Narrowed (skeptics):** Downstream-only: DDK never binds FRSRCP, and DDK's languages use Xtext's default provider, which has the same live iteration. It matters for consumer products that bind FRSRCP. A new search sets the input before its job is scheduled, so only re-showing a still-running search races (history, page switcher, reopened or second Search view). **Severity rationale:** lowered from medium to low: downstream-only and behind a user action during a running search. +- **Trace:** UI switches back to running R: `inputChanged` → `addListener` → iterator; search thread `accept()` → `matchingReferences.add` → iterator `next()` throws CME. +- **Models:** `formal/find-refs/tla/NOTES.md`, `formal/find-refs/lean/NOTES.md`, `formal/find-refs/tla/traces/orig_NoCME.txt`, `formal/find-refs/fix-cme.patch`. +- **Test:** `FRSRCPT#inputChangedToleratesReferencesAcceptedWhileRepopulating`, enabled on master by [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552). It failed on master before the fix and passes with it. + +#### REF-3 — Lost update: UIUpdater clears `isUIUpdateScheduled` after its unlocked `isEmpty()` check + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase B 3/3 upheld, 1 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs F1 (CONFIRMED), Lean round 2 refs F1 (CONFIRMED); javap of upstream `UIUpdater` shows it clears the flag first. +- **Locations:** FRSRCP:217, FRSRCP:220, FRSRCP:173. +- **Claim:** The UIUpdater drains, refreshes, checks `isEmpty()` (FRSRCP:217), and only then clears the flag (FRSRCP:220). A node added in between sees the flag still true and schedules nothing; if it was the last match, it is never shown. +- **Narrowed (skeptics):** The window also covers a new child of an already-expanded root added during `refresh()` (:216). Display only: the model and count are correct, and re-running recovers. **Severity rationale:** kept at medium against the skeptics' low, because the trigger is an ordinary search, not a narrow path; the impact (a missing result) is recoverable, which is the medium definition. +- **Trace:** UI drains [n1], refreshes, `isEmpty()`=true → search thread adds n2, reads flag==true, does not schedule → UI sets flag=false → search ends; n2 never shown. +- **Models:** `formal/find-refs/tla/NOTES.md`, `formal/find-refs/lean/NOTES.md`, `formal/find-refs/tla/traces/orig_NoLostRoot.txt`, `formal/find-refs/tla/traces/orig_NoLostRef.txt`. +- **Test:** proposed (REF-PR2): deterministic `isEmpty()` injection via an `ArrayList` subclass that delivers Added. + +#### REF-4 — `resourceNode` check-then-act creates duplicate root nodes and loses references + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 2 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs F4 (CONFIRMED), Lean round 2 refs F5 (CONFIRMED). +- **Locations:** FRSRCP:158, FRSRCP:161. +- **Claim:** `resourceNode` (FRSRCP:157-166) does an unguarded get-then-put on `rootNodes`. The UI thread (`inputChanged` repopulation) and the search thread's Added handler call it concurrently; the last put wins and references on the orphaned node disappear. DDK-only. In Lean this was masked by REF-3 until REF-3 was patched. +- **Narrowed (skeptics):** Reachable only when a still-running search is re-shown from history, and today the same interleaving nearly always throws the REF-2 CME first, so REF-4 is an independent defect only after REF-2 is fixed. A lost reference returns on the next `setInput`. Any fix lock must be ordered against `listeners`. **Severity rationale:** lowered from medium; reachability is limited to history re-show and masked by REF-2. +- **Trace:** UI `addReference(r1)`: `get(u1)`=null; search thread Added(r2): `get(u1)`=null; both put, last wins, r1 hangs under the orphan. +- **Models:** `formal/find-refs/tla/NOTES.md`, `formal/find-refs/lean/NOTES.md`, `formal/find-refs/tla/traces/orig_OneRootCreated.txt`, `formal/find-refs/tla/traces/orig_flagfixed_lost_ref_via_duplicate_root.txt`. +- **Test:** proposed (REF-PR4): state-polled two-thread test via a forwarding `rootNodes` map; fails on master. + +#### REF-5 — A running search's node appears in another search's view (clear before `removeListener`) + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase B 3/3 upheld, 0 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs F3 (CONFIRMED), Lean round 2 refs F3 (CONFIRMED). +- **Locations:** FRSRCP:111, FRSRCP:113. +- **Claim:** `inputChanged` clears `rootNodes` (:111) before `removeListener` (:113). An in-flight Added handler of the old search writes into the cleared map, which now belongs to the new input. The leak persists when the new input is a search picked from history, and is transient for a fresh search. `batchAddNodes` is not cleared on input change either. **Upstream variant (REF judge graft 8):** upstream also clears before `removeListener` and never clears its equivalent `batchedSearchResultEvents` on `setInput`; that part goes in the upstream issue, and REF-PR3 clears DDK's `batchAddNodes`. +- **Skeptic notes:** Swapping the two lines is correct only because `fireEvent` holds the listeners monitor, so `removeListener` waits for in-flight handlers. **Severity rationale:** kept at medium; not called overstated, and reachable whenever the user switches searches while one is running. +- **Trace:** search thread `accept(ref0)`: `get(uri0)`=null → UI `rootNodes.clear()` → search thread puts node0, schedules → UI `A.removeListener` → `setInput(B)` refresh shows A's node0. +- **Models:** `formal/find-refs/tla/NOTES.md`, `formal/find-refs/lean/NOTES.md`, `formal/find-refs/tla/traces/orig_NoForeign.txt`. +- **Test:** proposed (REF-PR3): single-threaded Mockito test. + +#### REF-6 — The same reference is shown twice + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs F5 (CONFIRMED), Lean round 2 refs F5 (part). +- **Locations:** FRSRCP:116, FRSRCP:119. +- **Claim:** `accept()` adds to the list outside the lock and fires Added afterwards. If `inputChanged` registers and iterates in between, the reference is added from both the snapshot and the event. Nothing dedupes. +- **Narrowed (skeptics):** Only when a still-running query is re-shown; the window is a few instructions wide, and a CME (REF-2) is more likely. Cosmetic; cleared on rerun. Dedup must use a lock taken by `addReference`, not the viewer lock. +- **Trace:** search `add(r2)` → UI `addListener`, iterate, `addReference(r2)` → search `fireEvent(Added r2)` → `addReference(r2)` again. +- **Models:** `formal/find-refs/tla/NOTES.md`, `formal/find-refs/lean/NOTES.md`, `formal/find-refs/tla/traces/orig_NoDupRef.txt`. +- **Test:** proposed (REF-PR4): an `addListener` stub fires Added for a reference already in the list. + +#### REF-7 — Tree node `children` list read and written concurrently without synchronisation + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 2 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs, code-reading extra (no verdict, not modelled). +- **Locations:** FRSRCP:137. +- **Claim:** `ReferenceSearchViewTreeNode.children` is a plain `ArrayList`. The search thread writes it via `addChild`; the UI thread reads it in `getChildren`/`hasChildren` and during expansion. DDK-only (the TLA+ note says the race exists upstream; the REF judge found upstream mutates nodes only on the UI thread). +- **Narrowed (skeptics):** `getChildren`/`hasChildren` use `toArray`/`isEmpty` and cannot throw a CME; at worst they read stale or torn state. The real CME risk is the for-each at :243 with `removeChild` at :255 in `descriptionsChanged` during an active search. +- **Models:** `formal/find-refs/tla/NOTES.md` ("Code-reading extras"). +- **Test:** proposed (REF-PR6): a seam that pauses a children read while another thread adds. + +#### REF-8 — `descriptionsChanged` mutates `rootNodes` and children on the UI thread concurrently with the search thread + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 refs, code-reading extra (no verdict, not modelled). +- **Locations:** FRSRCP:228. +- **Claim:** `descriptionsChanged` (FRSRCP:228-270) mutates `rootNodes` and node children while the search thread adds. Same class as REF-4. DDK-only. +- **Narrowed (skeptics):** `rootNodes` (a `ConcurrentMap`) operations are individually safe. The risk is the children `ArrayList`: a CME, or a match orphaned between `isEmpty()` (:257) and removal (:258). Needs a build delta during an active search; transient. +- **Models:** `formal/find-refs/tla/NOTES.md` ("Code-reading extras"). +- **Test:** proposed (REF-PR6): interleave Added into the `descriptionsChanged` check-then-act. + +#### REF-9 — An exception in `UIUpdater.runInUIThread` leaves `isUIUpdateScheduled` stuck true + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** Lean round 2 refs, observation (no verdict, not modelled). +- **Locations:** FRSRCP:207, FRSRCP:220. +- **Claim:** If `runInUIThread` throws before clearing the flag, that provider instance never schedules another update. +- **Narrowed (skeptics):** The disposed-viewer example is harmless (the provider is disposed with its viewer). Realistic triggers are a `RuntimeException` from a label provider or comparator on a live viewer, or the UIJob being cancelled before it runs. The page reuses the provider, so later searches in that page are affected until `setInput` or the view is recreated. +- **Models:** `formal/find-refs/lean/NOTES.md` ("Observations"). +- **Test:** proposed (REF-PR2): a mocked viewer whose `refresh()` throws once. + +#### REF judge + +- **Winner:** Plan A. It implements the design that was model-checked (TLA fixed set {flag, detach, dedupe, snapshot, plock, reset}, Lean `Fixed.lean`), and each PR maps to one ablation row, so every intermediate state is model-covered: after PR1-3 the code equals Lean 'lost+order+snap', after PR4 TLA `MC_no_reset`. Plan B's central PR confines the tree to the UI thread with a FIFO queue, which is not model-checked, bundles REF-1/4/7/8 into one rewrite, and falls back to Plan A. Plan A also has more failing-first tests (deterministic REF-3 injection; a REF-4 two-thread test that fails on master). +- **Facts verified by the judge:** FRSRCP:217-220 is the lost-update window; upstream (e8855b27fd) clears the flag first (`ReferenceSearchResultContentProvider:205`); the facts behind REF-A1 and REF-A2 ([4.3](#43-verified-observations)). +- **Correction:** the REF-2 null-slot mechanism (recorded under REF-2). +- **Grafts from Plan B:** REF-9 test via a throwing `refresh()`; a disposed-viewer guard in UIUpdater and `descriptionsChanged`; the REF-1 test through a real `ReferenceSearchResult.reset()`; the REF-6 test shape; REF-A1/REF-A2 as their own optional PR; UI-thread confinement as a follow-up gated on a `confine` model variant; one bundled Xtext issue. +- **Weakest part:** REF-7/REF-8: the fix is residual, argued by analogy, and not modelled. + +#### REF fix plans + +All REF PRs touch FRSRCP and `FRSRCPT`. **Version bump:** `com.avaloq.tools.ddk.xtext.ui` 17.3.3 → 17.3.4 (MANIFEST + pom), done only by the first REF PR merged in the cycle. `xtext.ui.test`: none (not in the p2 baseline). + +**REF-PR1 (REF-2)** + +- **Fix:** In `inputChanged` (FRSRCP:118), iterate `Lists.newArrayList(getMatchingReferences())` and skip null elements (`fix-cme.patch` plus a null guard, needed for the reason recorded under REF-2; without it `getContainerEObjectURI()` NPEs). +- **Files:** FRSRCP, `com.avaloq.tools.ddk.xtext.ui/META-INF/MANIFEST.MF`, `com.avaloq.tools.ddk.xtext.ui/pom.xml`, `FRSRCPT`, `com.avaloq.tools.ddk.xtext.ui.test/src/com/avaloq/tools/ddk/xtext/ui/test/XtextUiTestSuite.java`. +- **Test strategy:** Enable `inputChangedToleratesReferencesAcceptedWhileRepopulating`. Add a null-guard case where `toArray()` returns `{ref, null}`: no exception, exactly one root. +- **Risks:** Very low: an O(n) copy per view switch. A reference missed in the window arrives later via its Added event; duplicates in the window are REF-6. +- **Model evidence:** TLA `MC_no_snapshot` violates NoCME; the original model violates it in 19 steps. Lean P4 fails in 8 steps on 'as written' and 'lost', holds on 'lost+order+snap'. +- **Upstream:** Shared (Xtext `inputChanged` :135-138). Bundled Xtext issue with REF-6 and the REF-5 upstream variant, posted only with the user's approval. + +**REF-PR2 (REF-3, REF-9)** + +- **Fix:** In `UIUpdater.runInUIThread` (FRSRCP:205-223), make `isUIUpdateScheduled = false` the first statement, before the synchronized drain. After the drain, return OK if the viewer or its control is null or disposed. Delete the unlocked `isEmpty()`/`schedule(250)` tail and `JOB_RESCHEDULE_DELAY`. This is upstream's order: the volatile clear happens-before the drain's unlock, so a concurrent add is either drained or sees `false` and reschedules, and an exception can no longer leave the flag stuck. Fallback if throttling matters: Lean F1 option B, an atomic test-and-set under the batch lock. +- **Test strategy:** REF-3: replace the final `batchAddNodes` field by reflection with an `ArrayList` subclass whose `isEmpty()` fires Added inline; invoke `runInUIThread` on a reflectively built UIUpdater; assert `!batchAddNodes.isEmpty() ⇒ isUIUpdateScheduled`. Companion case injects from the `refresh()` answer. REF-9: a mocked viewer's `refresh()` throws on its first call; fire Added(a), then Added(b) with `useUIThread=false`; `verify(viewer, timeout(5000).times(2)).refresh()`. If reflection is rejected, add a package-private `updateViewer(IProgressMonitor)` seam. +- **Risks:** Drops the 250 ms throttle, so full refreshes can run back to back on large searches (upstream accepted this). +- **Model evidence:** The TLA fixed model uses exactly this order. `MC_no_flag` violates NoLostRoot; the original model violates NoLostRoot and NoLostRef in 31 steps. Lean 'lost' turns P1' green; `Proof.no_lost_root` proves option B for all sizes. REF-9 is not modelled; it is removed by construction. +- **Upstream:** Not shared; the DDK copy regressed upstream's order. + +**REF-PR3 (REF-5)** + +- **Fix:** Move `removeListener(this)` on the old input above `rootNodes.clear()`, and clear `batchAddNodes` under its monitor. +- **Test strategy:** Single-threaded Mockito: A's `removeListener` stub fires Added(A, refA). Call `inputChanged(viewer, null, A)`, then `inputChanged(viewer, A, B)` with B empty; assert `getElements(B).length == 0`. +- **Risks:** Minimal; no new lock ordering. +- **Model evidence:** TLA `MC_no_detach` violates NoForeign; the original model violates it in 23 steps. Lean P3 fails in 12 steps ('as written') and 10 ('lost'), holds with 'order'. +- **Upstream:** The variant recorded under REF-5 is shared; bundled issue, subject to approval. + +**REF-PR4 (REF-4, REF-6)** + +- **Fix:** Add a private `lock`, documented never to be held while taking `listeners` or the viewer monitor or while waiting on the UI, and an identity `Set addedReferences` guarded by it. The Added handler does `addReference` and the flag test-and-set under `lock`; `addReference` returns early for an already-added reference. `inputChanged` does `removeListener`, `addListener` and the viewer set outside the lock, then clears, snapshots and repopulates under it. The null-input path and `dispose()` clear under the lock. Lock order: search thread `listeners → lock`, UI `viewer → lock`; no cycle. +- **Test strategy:** REF-6: A's `addListener` stub fires Added(A, r) for an r already in the list; assert exactly one child. REF-4: replace `rootNodes` by reflection with a forwarding map whose `get` starts thread T delivering Added for the same URI, then polls T's state (10 s cap); assert one root and both references reachable. +- **Risks:** The new monitor is the main risk for future edits. Dedup is by identity, so equal-but-distinct descriptions still show twice, as today. +- **Model evidence:** TLA `MC_no_plock` violates OneRootCreated; `MC_no_dedupe` violates NoDupRef (46-step trace); `computeIfAbsent` is redundant under plock and fails NoStale without it. Lean 'lost+order+snap' still fails P1 in 25 steps (REF-4). After this PR the code equals TLA `MC_no_reset`, which fails only NoDeadlock. +- **Upstream:** REF-6 shared; REF-4 DDK-only. + +**REF-PR5 (REF-1)** + +- **Fix:** In the Reset branch (FRSRCP:177-189), remove `syncExec`. On the search thread, clear `rootNodes`, `addedReferences` and `batchAddNodes` under `lock`, then `asyncExec` a `refresh()` + `expandToLevel(1)` guarded by a non-disposed control, without taking the viewer monitor. +- **Test strategy:** Park the UI thread with an `asyncExec` runnable awaiting a `release` latch. Thread S runs `result.reset()` on a real `ReferenceSearchResult` subclass, so `listeners` is genuinely held. Assert S finishes within 10 s; a finally block releases and joins S. +- **Risks:** Must land with or after REF-PR4: `asyncExec` alone loses post-reset nodes (Lean '+async' P1, 13 steps), and unlocked clearing shows stale nodes (NoStale). Old rows stay visible until the async refresh runs. +- **Model evidence:** TLA `MC_no_reset` violates NoDeadlock (6 steps). The full fixed model passes every invariant, including `<>`/3 runs/4 switches (50.5M states). Lean 'fixed' is green at both bounds; the planted bug is caught in 2 steps. +- **Upstream:** DDK-only; upstream batches Reset in the UIUpdater. + +**REF-PR6 (REF-7, REF-8)** + +- **Fix:** Reuse the REF-PR4 lock around the `getChildren`/`hasChildren` reads and the `descriptionsChanged` body, plus a null/disposed viewer guard. Residual: `collectReferenceDescriptions` and `expandToLevel` still read outside the lock. The complete alternative is Plan B's UI-thread confinement, as a follow-up gated on a `confine` variant passing in `FindRefs.tla` and `Fixed.lean`. +- **Files:** FRSRCP, `FRSRCPT`, `formal/find-refs/tla/FindRefs.tla`. +- **Test strategy:** First extend `FindRefs.tla` with a `DescriptionsChanged` action and a modCount-tracking iterator. Then Java seams: pause a children read while T adds (REF-7); interleave Added into the `descriptionsChanged` check-then-act (REF-8). If the seams get too invasive, ship on the model extension plus review, and say so in the PR. +- **Risks:** Brief contention between expansion and the Added handler. +- **Model evidence:** None yet; argued by analogy with plock. +- **Upstream:** DDK-only. + +**REF-PR7 (optional: REF-A1, REF-A2)** + +- **Fix:** (REF-A1) `dispose()` clears under the lock, then calls `super.dispose()` so `detachListenerFromIndex` runs; drop the constructor's duplicate `addListener` (FRSRCP:69-71). (REF-A2) Override `remove(ReferenceSearchViewTreeNode...)` under the lock: remove a root from `rootNodes`, otherwise `parent.removeChild`, and drop dedupe entries. +- **Test strategy:** (REF-A1) A mock `IResourceDescriptions` implementing `Event.Source`: `removeListener` after `dispose()`, exactly one `addListener` after construction. (REF-A2) Populate one reference, call `remove(root)`, expect an NPE on master. Both must be red on master before the PR claims them (phase C confirmed REF-A2 from bytecode; the NPE has not been run). +- **Risks:** Low. The index listener set is a `CopyOnWriteArraySet`, so the duplicate `addListener` is redundant, not a double notification. +- **Model evidence:** None; code reading of Xtext `ReferenceSearchViewPageActions:150` and `ReferenceSearchResultContentProvider:104-112`, :295. Not an upstream bug. + +### 3.3 STO — binary model storage (`MonitoredClusteringBuilderState`) + +**Reachability:** DDK never enables binary storage: both of its target platforms return a null `IBinaryModelStore` (see [rule 2](#1-method-and-legend)). Every STO finding affects only downstream platforms that supply a store and bind DLRSF, so STO severities are capped at medium. + +#### STO-1 — URI removed from sources before its binary is written (MCBS:656) + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 storage B1 (CONFIRMED), Lean round 2 storage B2 (CONFIRMED). +- **Locations:** MCBS:656, MCBS:654, MCBS:754, MCBS:596. +- **Claim:** `storeBinaryResource` only submits the store (MCBS:654/726), but MCBS:656 removes the URI from the source-level set immediately; the worker's removal after `saveResource` (MCBS:753-754) is the correct one. The load-failure path (changedURI set, resource null) also removes a URI for which no store happens, so the previous build's binary becomes loadable. Upstream never removes URIs from this set. STO-1 enables STO-3 and STO-4. Fix: delete MCBS:656. +- **Narrowed (skeptics):** The main resource set is unaffected: the resource stays in memory, and `clearResourceSet` awaits stores first. Stale or partial reads happen only via (a) parallel-loader child sets loading dependencies while stores are pending, (b) stores dropped after the 1-minute await timeout, and (c) the load-failure path, where the delete delta hides the resource from index-based scoping so only direct URI proxies reach the stale binary. **Severity rationale:** lowered from high to medium. It is downstream-only (cap), and the stale-binary read needs a timeout, a load failure or a same-cluster child-set load. It stays above the skeptics' low because it is the root cause of STO-3/STO-4 and its effect is a silently wrong model. +- **Trace:** `next()` returns a → link → submit store → `getSources().remove(a)` while the store is queued. Load-failure variant: resource null, no store, URI still removed → old binary loadable. +- **Models:** `formal/binary-storage/tla/NOTES.md`, `formal/binary-storage/lean/NOTES.md`. +- **Test:** proposed (STO-PR1, on the STO-PR0 harness): three failing-first cases. + +#### STO-2 — Data race on the sources `HashSet` + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 storage B5 (CONFIRMED), Lean round 2 storage B1 (CONFIRMED); javap of `SourceLevelURICache.` shows a plain `HashSet`. The corruption outcomes were not demonstrated in Java. +- **Locations:** MCBS:656, MCBS:754, MCBS:1296, MCBS:1536, PRL:168. +- **Claim:** The set is mutated without synchronisation by the builder (MCBS:656, 1296-1298) and up to 4 storage workers (MCBS:754), and read by loader jobs (PRL:168-174). `update()` is synchronized, but only the builder takes that lock. A lost add makes a queued URI load from its stale binary. +- **Narrowed (skeptics):** The only harmful interleaving is a worker remove racing a builder add or resize in `queueAffectedResources`, in a very narrow window. Other interleavings cost at most an extra parse. One skeptic says a `contains()` miss during a resize cannot happen, because adds run while no load operation is live. The worker remove at 754 duplicates 656. **Severity rationale:** lowered from high to low: downstream-only, one narrow harmful interleaving. +- **Trace:** builder `remove(a)` at 656 overlaps worker remove at 754 (`orig_SetThreadSafe`); builder `add(b)` at 1298 overlaps worker remove (`orig_adr`); loader `contains()` overlaps worker remove (`orig_rdw`). +- **Models:** `formal/binary-storage/tla/NOTES.md`, `formal/binary-storage/lean/NOTES.md`. +- **Test:** proposed (STO-PR2): lock-contract tests, no stress test. + +#### STO-3 — Loaders start before storage is awaited and read partial or stale binaries of dependencies + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 1 overstated; suggested severity medium, medium, low. **Origin:** TLA+ round 2 storage B2 (CONFIRMED, given that loading u also loads its dependencies), Lean round 2 storage B3 (CONFIRMED ordering; impact language-dependent). +- **Locations:** MCBS:668, MCBS:673, MCBS:1204. +- **Claim:** MCBS:668-669 starts the next cluster's loader before `clearResourceSet` awaits storage (MCBS:673 → 1204). Combined with STO-1, a loader that pulls in a dependency still being stored reads a half-written or previous-build binary; this also happens within one cluster. A truncated read falls back to parsing (`StorageAwareResource.load` catches the `IOException`), but a stale binary is accepted silently. +- **Narrowed (skeptics):** The root cause is STO-1: reordering the await would not fix it, deleting 656 does. A lasting wrong result needs a language whose `getResource(uri, true)` copies values from other resources during load, and no DDK language does; PRL unloads pulled-in dependencies, and references are re-resolved on the main thread after the await. **Severity rationale:** lowered from medium to low: downstream-only and conditional on a language shape DDK does not have. Its fix is STO-1's. +- **Trace:** cluster 1 stores a and removes it from sources → `queueAffectedResources` adds b → `load(queue)` starts b before the await → b's load pulls a from its partial or stale binary. +- **Models:** `formal/binary-storage/tla/NOTES.md`, `formal/binary-storage/lean/NOTES.md`. +- **Test:** proposed (STO-PR1 case c): within-cluster load sees `contains(a)` while a's store is latched. + +#### STO-4 — Stores dropped by `shutdownNow` after an await timeout leave stale binaries that persist across builds + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase B 3/3 upheld, 2 overstated; suggested severity medium ×3. **Origin:** Lean round 2 storage B4 (CONFIRMED), TLA+ round 2 storage B3 (part). +- **Locations:** MCBS:804, MCBS:879, MCBS:880, MCBS:1524. +- **Claim:** The storage await defaults to 1 minute with 0 retries (MCBS:804). On timeout or interrupt, `shutdownNow` (MCBS:879) drops queued stores and logs only their count. Their URIs were already removed at 656, so this build and later builds (which install only `toBeUpdated` as sources, MCBS:1524-1536) load outdated binaries until the resource changes. +- **Skeptic notes:** It can happen at any mid-build `clearResourceSet` boundary, not only at the end. A resource with no earlier binary falls back to source. `deleteBinaryResources` covers only `toBeDeleted` (L453). The interrupt path practically never fires. **Severity rationale:** medium (cap): wrong models that persist across builds, behind a 1-minute storage backlog on a downstream platform. +- **Trace:** a and c stored and removed from sources → await times out → `shutdownNow` drops c → next cluster resolves a/c from outdated binaries; c's old binary persists. +- **Models:** `formal/binary-storage/lean/NOTES.md`, `formal/binary-storage/tla/NOTES.md`. +- **Test:** proposed (STO-PR3): 10 ms await must delete the dropped stores' binaries. + +#### STO-5 — Running stores are neither awaited nor reported after `shutdownNow` + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 storage B3 (CONFIRMED), Lean round 2 storage B5 (CONFIRMED). +- **Locations:** MCBS:837, MCBS:871, MCBS:1203, MCBS:1207, MCBS:684. +- **Claim:** `terminateBinaryStorageExecutor` never waits after `shutdownNow`. Running tasks continue while the builder recreates the executor (MCBS:871), clears the resource set they serialise (MCBS:1207; the comment at 1203 names this hazard), and continues. The builder can open a partial binary. +- **Narrowed (skeptics):** They disagree on the mechanism: one says the default Xtext `PortableURIs` NPEs on a detached resource, after which DLRSF deletes the binary; another says DDK's write path (`DirectLinkingPortableURIs`, `DirectLinkingResourceStorageWritable`) does not read the resource set, so there is no NPE. All agree the realistic harm is a narrow same-URI race: loading the previous-build binary, or catching the single final `generateFile` write part-way. **Severity rationale:** lowered from medium to low: downstream-only, behind the same timeout as STO-4, and a narrow same-URI window. +- **Trace:** a stored async and removed → await times out → `shutdownNow` (running task continues) → new executor, `clearResourceSet` → cluster 2 loads a from its partial binary (`orig_mainpartial`, `orig_NoDetachedStore`). +- **Models:** `formal/binary-storage/tla/NOTES.md`, `formal/binary-storage/lean/NOTES.md`. +- **Test:** proposed (STO-PR3): after the await returns, all running stores' completion flags are true (ordering-flag assertion). + +#### STO-6 — After a link exception the resource is detached but still stored + +- **Severity / status:** low / PLAUSIBLE. **Verification:** phase B 1/3 upheld, 3 overstated; suggested severity low, none, none. **Origin:** TLA+ round 2 storage B4 (CONFIRMED, low). +- **Locations:** MCBS:608, MCBS:654. +- **Claim (as raised):** When linking throws after `addResource`, the outer catch removes the resource (MCBS:608) but execution falls through to `storeBinaryResource` (MCBS:654); an NPE follows, DLRSF deletes the storage, and a valid binary is lost. +- **Why only PLAUSIBLE:** All three agree the store of a detached resource happens; they reject the "valid binary lost" part. + - Skeptic 1 (upheld, narrowed): an NPE (`PortableURIs.toPortableURI` on the null resource set) occurs only if the resource references anything outside itself. The deletion that follows is harmless: the catch registers a delete delta (MCBS:610-612), and the binary was stale anyway because the resource was being rebuilt. The real effect is a misleading ERROR log. A resource with no cross-resource references serialises without an NPE, writing a binary of a resource whose linking threw. + - Skeptic 2 (refuted): DDK's save path never touches `getResourceSet`, so there is no NPE. If linking recorded errors, DLRSF deletes the stale binary by design (DLRSF:66 Javadoc, 77-79); otherwise a fresh binary is written. That is an overwrite, not a loss. For Check (`BatchLinkableResourceStorageFacade`), `getSourceContainerURI` NPEs, the delete NPEs too, and the old binary is kept. + - Skeptic 3 (refuted): serialisation does not need the resource set; no NPE, no data loss. Keeping the binary, as a skip-only fix would, is arguably worse than deleting it. +- **Trace:** `orig_NoDetachedStore` with AllowLinkFail: exception → `resources.remove` (608) → `storeBinaryResource` (654). +- **Models:** `formal/binary-storage/tla/NOTES.md`. +- **Test:** proposed (STO-PR4). + +#### STO-7 — Swallowed `IOException` in `writeResource` keeps the old binary while the URI leaves sources + +- **Severity / status:** — / REFUTED. **Verification:** phase B 0/3 upheld the claim as raised: 2 refuted it outright; the third did not formally refute, but upheld only a different claim (a non-DLRSF facade, practically unreachable, now STO-A4) and said the finding "frames the problem around DLRSF, where it cannot occur". 2 overstated. **Origin:** Lean round 2 storage B6 (CONFIRMED code path, rare trigger). +- **Locations:** MCBS:754; RSF:97-103. +- **Claim (as raised):** RSF catches the `IOException` from `writeResource` and keeps the old file; MCBS:754 then removes the URI. +- **Why refuted:** For DLRSF, `DirectLinkingResourceStorageWritable.writeEntries`/`writeNodeModel` wrap every `IOException` and `RuntimeException` in `WrappedException` (128-134, 145-146), so RSF's `catch (IOException)` is never reached. DLRSF catches it, deletes the old storage and rethrows (77-80); MCBS:761 logs it and 754 is skipped. The only `IOException` left for RSF to swallow would come from `ZipOutputStream.close()` into an in-memory `ByteArrayOutputStream`, which does not throw. The location is also wrong: the removal that matters is the unconditional one at MCBS:656 (STO-1), not 754. The residual gap belongs to a different facade (STO-A4). Lean `ioFail` models RSF's swallow but not DDK's wrapping. +- **Models:** `formal/binary-storage/lean/NOTES.md`. +- **Test:** none. STO-PR0 may add a characterization test that documents the refutation. + +#### STO-8 — `InterruptedException` during the storage await swallowed without restoring the flag + +- **Severity / status:** low / CONFIRMED. **Verification:** phase B 3/3 upheld, 3 overstated; suggested severity low ×3. **Origin:** TLA+ round 2 storage, side note in B3 (no verdict). +- **Locations:** MCBS:865. +- **Claim:** The catch at MCBS:865-867 does not re-interrupt the thread. +- **Narrowed (skeptics):** Build cancellation uses `IProgressMonitor`, and nothing in DDK interrupts the builder thread. Latent hygiene, matching `ParallelResourceLoader`. +- **Models:** `formal/binary-storage/tla/NOTES.md` (B3). +- **Test:** proposed (STO-PR5): interrupt a helper that is TIMED_WAITING in the await. + +#### STO judge + +- **Winner:** Plan A. +- **Common ground:** Both plans delete MCBS:656, keep the load/await order at 667-674, skip the store and delete the binary after the outer catch (STO-6), drain running stores and delete dropped ones (STO-4/5), restore the interrupt after executor recreation (STO-8), and treat STO-7 as refuted (no fix). Line numbers checked against the worktree. +- **Rationale:** (1) Plan B's first PR extracts a `BinaryStorageCoordinator`: a production refactor of protected, downstream-overridden API that closes no finding, with tests that bypass `doUpdate` and the loader path. Plan A's STO-PR0 is test-only and exercises the real `doUpdate`, including STO-3's loader-side `contains()`. (2) For STO-2, Plan A's locked view preserves set identity with Xtext's `BuildContext` and is exactly the model-checked TLA FixB. Plan B's per-load-op snapshot is unmodelled and changes the live-view contract (Javadoc 1519-1522); its fallback `ConcurrentHashMap.newKeySet` breaks set identity. (3) Plan B's `ExecutorService.close()` waits in one-day chunks with no logging. Plan A also spells out that restoring the interrupt inside the drain loop would spin. +- **Grafts from Plan B:** `invalidateBinaryResoureCache` alongside `deleteBinaryResources`; an ordering-flag assertion for STO-5; drive STO-8 by interrupting a TIMED_WAITING helper; FixA alone leaves a racy-`contains` residue (Lean `raceNondet`, P3stale), so STO-2 lands before the executor PRs; builder confinement and `ConcurrentHashMap` kept as documented alternatives; the harness must call `updateBinaryStorageAvailability` with a non-null store, and end-to-end validation has to happen downstream; the STO-6 PR-body framing (the deleted binary is outdated; the defect is serialising a detached resource). Follow-ups it named are STO-A1 and STO-A3 ([4.3](#43-verified-observations)), plus an optional end-of-build ledger ("delete binaries of URIs processed but not stored") only if gaps remain. + +#### STO fix plans + +Test files named below live in `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/builder/`. **Version bump:** `com.avaloq.tools.ddk.xtext.builder` 17.3.1 → 17.3.2, by the first production PR on this bundle in the cycle (shared with LDR). `xtext.test`: none (already 17.3.2 against baseline 17.3.1). + +**STO-PR0 (test harness, no finding)** + +- A test subclass of MCBS in package `com.avaloq.tools.ddk.xtext.builder` inside `xtext.test`, constructed with a mock `IXtextTargetPlatformManager`, with `@Inject` members from a Guice module of mocks and fakes: `InMemoryFileSystemAccess`, a mock DLRSF, a null `IResourceDescription.Manager` (linking skipped, store still runs), and a fake `RESOURCELOADER_CROSS_LINKING` loader. It calls `updateBinaryStorageAvailability` with a non-null `IBinaryModelStore`. Seam: a latch-gated override of `doStoreBinaryResource`. Includes a positive control and optionally the STO-7 characterization test. Register in `XtextTestSuite`. No bump. + +**STO-PR1 (STO-1, STO-3)** + +- **Fix:** Delete MCBS:656 and nothing else. A URI then leaves sources only at MCBS:754, after a successful save (under `CallerRunsPolicy` the builder runs the same line). Do not reorder `load(queue)` and `clearResourceSet`: FixA alone closes the cross-cluster and within-cluster reads (TLA `orig_loaderpartial_M`), and the current order keeps storing and loading overlapped. +- **Files:** MCBS, builder MANIFEST/pom, `MonitoredClusteringBuilderStateStorageTest.java` (new), `XtextTestSuite.java`. +- **Test strategy:** All three fail on master. (a) Cluster {a,b}: while a's store is latched, `contains(a)` is true. (b) Load failure `LoadOperationException(a, IOException)`: `contains(a)` is true. (c) Within a cluster, b's fake load sees `contains(a)` true while a's store is latched. Control (d): after `doUpdate`, `contains(a)` is false. +- **Risks:** Performance only: a dependency with a pending store is parsed instead of read from a binary. With storage disabled, processed URIs stay sources for the rest of the build, as upstream. A downstream override of `storeBinaryResource`/`doStoreBinaryResource` that never removes the URI loses binary loads within the same build; say so in the PR body. The cross-build part of the load-failure variant is closed by STO-PR4. +- **Model evidence:** TLA `ablateA` violates NotSourceOnlyWhenStored; `fixed_S/M/M2/L/L2/R` pass P1, LoaderNoPartialRead, MainNoPartialRead and NoStaleRead. Original traces: `orig_P1_nofail`, `orig_NotSourceOnlyWhenStored`, `orig_LoaderNoPartialRead`, `orig_loaderpartial_M`, `orig_NoStaleRead`. Lean `Generic.fixed_safe` proves P1 and P3 for all sizes; `buggy_unsafe`, `bug_p1`, `bug_partial`, `bug_stale_no_timeout` fail on the original. +- **Upstream:** Not shared. + +**STO-PR2 (STO-2)** + +- **Fix:** One monitor, the Xtext-owned `HashSet`. In `installSourceLevelURIs` (MCBS:1536), install a private `LockedSourcesView extends AbstractSet`: `contains`/`size`/`isEmpty` under `synchronized (delegate)`, iteration on a copy, `add`/`remove` throw. PRL picks it up through the adapter unchanged. Wrap MCBS:754, 1298 and 1314 in `synchronized (sources)`; comment 452/1528 as pre-publication writes. Assumes STO-PR1 has landed (otherwise 656 needs the lock too). +- **Files:** MCBS, `MonitoredClusteringBuilderStateStorageTest.java`. +- **Test strategy:** Lock contract, no stress test. (a) While the test holds the lock, a helper calling `contains()` reaches BLOCKED. (b) A worker at 754 blocks while the lock is held and removes the URI after release. (c) `add()` on the view is rejected. +- **Risks:** Negligible uncontended lock cost. A future writer could forget the lock; a class comment mitigates. No nesting, so no deadlock risk. +- **Model evidence:** TLA `ablateB` violates SetThreadSafe, including in the happy environment; `orig_SetThreadSafe` (11 states), `orig_adr` (15), `orig_rdw` (19). Lean `bug_p4` fails in 7 steps; `raceNondet` reaches P3stale in 8. The view implements the model-checked FixB. +- **Upstream:** Not shared; comes from DDK's `setSourceLevelUrisWithoutCopy` plus the asynchronous store. + +**STO-PR3 (STO-4, STO-5)** + +- **Fix:** (1) Replace the lambda at MCBS:726 with an inner `BinaryStoreTask` exposing `getUri()`, so `shutdownNow` returns identifiable tasks. (2) After `shutdownNow`, `terminateBinaryStorageExecutor` logs the dropped count at WARN and the URIs at INFO (the list can reach 15,000 entries), then calls `deleteBinaryResources` and `invalidateBinaryResoureCache` on them. It then drains, `while (!isTerminated()) awaitTermination(...)`, warning the active count each round and remembering any interrupt. All before the executor is recreated (871) and before `clearResourceSet` (1207). (3) Do not restore the interrupt here (STO-PR5). Do not use `ExecutorService.close()`. +- **Files:** MCBS, `MonitoredClusteringBuilderStateExecutorTest.java` (new), `XtextTestSuite.java`. +- **Test strategy:** A blocking `doStoreBinaryResource` that sets a completion flag in finally; submit 6 stores (4 run, 2 queue). STO-4: an await with a 10 ms timeout must `deleteStorage(u5/u6)` and never u1-u4. STO-5: release the latch once `isShutdown()`; when the await returns, all u1-u4 completion flags are true. +- **Risks:** The drain is unbounded, so a hung store (e.g. a stalled NFS) blocks the build; the periodic WARN makes it visible. A cap would reopen STO-5; state the trade-off in the PR body. Deleting dropped binaries costs a re-parse in the next build. +- **Model evidence:** TLA `ablateC` violates NoDetachedStore; the fixed models pass NoDetachedStore, MainNoPartialRead and StoresAccounted (MaxTimeouts 1, 2), and WitnessTimeout is still reached. Original traces `orig_mainpartial` (26), `orig_NoDetachedStore` (19). Lean `bug_p2` (35 steps) and `bug_pEnd_timeout` (38) fail; `fixed_default`, `fixed_caller_runs`, `fixed_no_old_binary` pass. The cross-build deletion is backed by Lean only. +- **Upstream:** Not shared. + +**STO-PR4 (STO-6; optional: STO-A2)** + +- **Fix:** In the outer catch (MCBS:585-613), set `resource = null` after removing it from the set, so the cache clear (651) and the store (654) skip it. If `changedURI != null`, call `deleteBinaryResources` and `invalidateBinaryResoureCache` on it; this also closes the cross-build part of the STO-1 load-failure variant. Optional: the STO-A2 null guard in `deleteBinaryResources` (MCBS:782). +- **Test strategy:** (a) The manager's `getResourceDescription` throws: no store for a, and `deleteStorage(a)` is called. (b) A load failure deletes a pre-seeded binary. +- **Risks:** One synchronous delete, on the failure path only. Frame the PR as "do not serialise a detached resource", not as fixing a lost valid binary (status PLAUSIBLE). A timeout `LoadOperationException` has no URI, so nothing is deleted there. The half-linked store after a `StackOverflowError` is STO-A3 and not covered. +- **Model evidence:** TLA `ablateD` violates NoDetachedStore (8-state AllowLinkFail trace); TLA FixD is skip-only. The deletion is backed by Lean `fixed_no_old_binary` (P5). +- **Upstream:** Not shared. + +**STO-PR5 (STO-8)** + +- **Fix:** At the end of `awaitBinaryStorageExecutorTermination(int, TimeUnit, int)`, after the executor is recreated (MCBS:871-873), restore the interrupt if one was caught. Not inside the catch, or the drain loop spins. +- **Test strategy:** A helper TIMED_WAITING in a 1-minute await is interrupted, then the latch is released. The helper sees `isInterrupted()` true afterwards, and the store completed before the await returned. +- **Risks:** The next await in the same build takes the `shutdownNow` path; after STO-PR3 those drops are deleted and logged, so the cost is performance only. Must land after LDR-PR2 and STO-PR3. +- **Model evidence:** Code reading (TLA NOTES B3); `AwTimeout`, `ablateC` and the fixed models cover the drain it depends on. +- **Upstream:** Not shared. + +**STO-7: no fix.** Refuted because DDK's DLRSF writable wraps every `IOException`, so the swallowing catch in RSF is unreachable for DLRSF and the old binary is deleted, not kept (see STO-7). Upstream: latent in Xtext 2.44 `RSF.saveResource`, harmless there. Review guidance only: a future facade whose writable lets a raw `IOException` through would reopen the gap (STO-A4). + +### 3.4 TRIE — qualified-name lookup + +**Reachability:** `ContainerQuery`, `PrefixedContainerBasedScope` and `ContainerBasedScope` use `QualifiedNamePattern.create*` in production. `QualifiedNamePattern.createFromGlobs` has no production caller (only `QualifiedNamePatternTest` and `QNLFT`); `TreeSetLookup` and `shareValues=true` have no production user. So TRIE-2, which shows the same symptom class as TRIE-12/13/14, is high while the glob findings are low: the glob API is latent until a downstream caller uses it. + +All TRIE test methods except `PAEDLT` live in `QNLFT`. Their red/green results were confirmed under the Tycho aggregator: red on master; green with `fix-plan.patch` plus the `com.avaloq.tools.ddk.xtext` bump, except `testMaxCharPattern` (TRIE-9, by design). With the patch, 5 existing `QualifiedNamePatternTest` methods (`testPatternWithoutWildcard`, `testRegexpPatterns`, `testAllPattern`, `testRecursiveWildcardPattern`, `testQualifiedPrefixNamePattern`) fail because they assert the old `"!"` bound; the fix PRs must update them. `QNLFT` needs `Import-Package: com.avaloq.tools.ddk.caching` in the `xtext.test` manifest (added in this PR). + +#### TRIE-1 — Case-sensitive pattern queries in `PatternAwareEObjectDescriptionLookUp` always return nothing + +- **Severity / status:** high / CONFIRMED. **Verification:** phase A 3/3 upheld three times. **Origin:** trie-lean B8, trie-tla F2, ro-selfmatch #0 (all CONFIRMED). +- **Fix PR:** [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551) (merged) — PAEDL:63 matches against `input.getName()`; `PAEDLT` is on master with its tests enabled. +- **Locations:** PAEDL:63, QNP:263, `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/scoping/ContainerQuery.java:300`. +- **Claim:** PAEDL:63 calls `((QualifiedNamePattern) name).matches(name)`, testing the pattern against itself, and `matches` returns false for any `QualifiedNamePattern` argument (QNP:263-264). So `getExportedObjects(type, pattern, false)` is always empty (Lean `All.consumer_cs_empty`). This affects `ResourceDescription2`, `SimpleResourceDescription`, `FixedCopiedResourceDescription` and `FingerprintResourceDescription` for case-sensitive `ContainerQuery` name patterns. Present since the initial contribution, 4e2c12841. +- **Trace:** descriptions {Foo, FooBar, foobar, Other}; `create("Foo*")` case-sensitive → [] (expected [Foo, FooBar]); `create("Other")` → []. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`, `formal/trie/lean/fix-plan.patch`. +- **Test:** `PAEDLT#testCaseSensitivePatternMatchesCandidateNames` and `#testCaseSensitiveExactPattern` (enabled on master by [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551); the other 3 methods are guards that pass and run), and `QNLFT#testCaseSensitivePatternQuery` (`@Disabled`). All pass with the fix. + +#### TRIE-2 — Top-level `*` / `**` pattern uses upper bound `"!"` and misses almost every name + +- **Severity / status:** high / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B1, trie-tla F1. +- **Locations:** QNP:372, QNP:374, QNSTL:113, QNSTL:190, `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/scoping/PrefixedContainerBasedScope.java:69`. +- **Claim:** `upperExclusive()` returns `QualifiedName.create("!")` for a single-segment `*`/`**`, so every name starting with a character ≥ `!` (letters, digits, `_`, `$`) falls outside the range; both lookups return nothing (Lean `All.topStar_unsound`). Reachable in production via `ContainerQuery.Builder.name("*")` and `PrefixedContainerBasedScope` with an empty prefix. +- **Trace:** `put(("b"),v)`; `get(pattern("*"))`: `find(("!"))` returns node b, taken as the stop marker → []. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testTopLevelWildcardFindsSingleSegmentNames` (tree and `TreeSetLookup`) and `#testTopLevelRecursiveWildcardFindsAllNames`, both `@Disabled`. + +#### TRIE-3 — `'!'` is not the successor of a segment, so names with a trailing char below `'!'` leak into results + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean B2, trie-tla F3); the merged trie-tla F4 was 2/3. **Origin:** trie-lean B2, trie-tla F3, trie-tla F4 (sibling-loop manifestation). +- **Locations:** QNP:379, QNP:386, QNP:363, QNSTL:200, QNSTL:223, PAEDL:62. +- **Claim:** Appending `'!'` is not a successor: for any c < `'!'`, [s] < [s+c] < [s+"!"] (Lean `All.bang_not_successor`). Pattern `"a"` returns a stored `"a "`, and `"a.*"` returns descendants of a sibling `"a "`. Neither the trie nor the ignore-case consumer re-checks `matches()`. Appending `'\u0000'` is proved to be the exact successor. Low because it needs names containing whitespace or control characters. +- **Trace:** `put(("a "),v)`; `get(pattern("a"))` → [v]. `put(("a ","x"),1)`, `put(("a","b"),2)`; `get(pattern("a","*"))` → [2,1]. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testExactPatternExcludesNameWithLowCharSuffix`, `#testChildWildcardExcludesSiblingWithLowCharSuffix` (`@Disabled`). + +#### TRIE-4 — A wildcard-free pattern's `matches()` accepts longer names + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B3, trie-tla F5. +- **Locations:** QNP:304, QNP:278, `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/scoping/ContainerBasedScope.java:86`. +- **Claim:** The non-glob loop never compares segment counts, so `pattern("a").matches(("a","b"))` is true (Lean `All.exact_matches_extensions`) while the lookups return only `("a")`. `ContainerBasedScope` filters with `matches()` and so accepts longer names for programmatic wildcard-free pattern criteria. +- **Trace:** store {a:0, a.b:1}; `get(pattern("a"))` = [0], but `matches` accepts a.b. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testExactPatternMatchesOnlyEqualLength` (`@Disabled`). + +#### TRIE-5 — Statistics size over-counts on duplicate put/putAll + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B5, trie-tla F6. +- **Locations:** QNSTL:590, QNSTL:601, QNSTL:639, `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/util/ArrayUtils.java:83`. +- **Claim:** `put()` does `size++` unconditionally and `putAll` adds `values.size()`, even when the value is already present. After a remove, statistics report entries for an empty lookup, and `initializeFrom`'s `size>0` guard (:639) then refuses it. +- **Trace:** `put(a,v)` twice → size=2; `remove(a,v)` → size=1, `get(a)`=null. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testSizeCountsMappingsOnce` (`@Disabled`). + +#### TRIE-6 — `putAll` stores duplicate values, so a single remove leaves the value mapped + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-tla). **Origin:** trie-tla F7. +- **Locations:** QNSTL:263, `ArrayUtils.java:79`, `ArrayUtils.java:83`. +- **Claim:** A new node takes `newValues` as-is (:263/:436). `ArrayUtils.addAll` returns the values unchanged for a null or empty target, and dedupes against the original array instead of the growing result. `putAll(n,[v,v])` stores v twice, and one remove leaves it mapped. `TreeSetLookup` shares `ArrayUtils` and behaves the same. +- **Trace:** TLC `op_exact`: `putAll(a,[v1,v1])`; `remove(a,v1)` → `get(a)`=[v1]. +- **Models:** `formal/trie/tla/NOTES.md`. +- **Test:** proposed (TRIE-PR6 cases b-d). Not in `QNLFT`, and `fix-plan.patch`'s size fix does not address it. + +#### TRIE-7 — `getMappings` drops blank intermediate segments + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B6, trie-tla F10. +- **Locations:** QNSTL:295. +- **Claim:** `!segment.isBlank()` is meant to skip the root but also skips real empty or whitespace segments, so `(" ","a")` comes back as `("a")`. +- **Trace:** `put((" ","a"),v)`; `getMappings(v)` → [("a")]; `TreeSetLookup` → [(" ","a")]. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testGetMappingsKeepsBlankSegments` (`@Disabled`). + +#### TRIE-8 — With `shareValues=true`, `get(pattern, false)` collapses multiplicities + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B7, trie-tla F8. +- **Locations:** QNSTL:406, QNSTL:436. +- **Claim:** `ValueSharingSegmentNode.matches` collects arrays in an identity-comparing `HashSet`, so a shared array is counted once while distinct equal arrays are counted each time. `shareValues=true` has no production caller in this repo. +- **Trace:** `shareValues=true`: `put(a,v)`, `put(a.b,v)`; `get(pattern("a**"), false)` → [v]; the reference gives [v, v]. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testValueSharingKeepsMultiplicity` (`@Disabled`). + +#### TRIE-9 — U+FFFF wraps the upper bound and collides with the tree's sentinel + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B9, trie-tla F13. +- **Locations:** QNP:389, QNP:367, QNSTL:520, QNSTL:174. +- **Claim:** `(char)(c+1)` wraps U+FFFF to U+0000. `TreeSetLookup.get` then throws `IllegalArgumentException` (fromKey > toKey), and the trie never meets the bound. A stored name `"￿"` merges into the sentinel node. A fix needs a design decision. +- **Trace:** `new TreeSetLookup().get(pattern("￿*"), false)` → IAE; tree `put("￿", m)`: `get(pattern("￿"))` → [], but `get(name)` → [m]. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testMaxCharPattern` (`@Disabled`). It still fails with `fix-plan.patch` by design: the patch and the Lean proofs assume no U+FFFF in names (assumption A1). It stays disabled until TRIE-PR10 is decided. + +#### TRIE-10 — `TreeSetLookup` returns spurious names for patterns with an empty segment + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean), 3/3 (trie-tla). **Origin:** trie-lean B4, trie-tla F14. +- **Locations:** QNP:224, QNP:340, `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/TreeSetLookup.java:66`. +- **Claim:** `compareTo` returns -1 as soon as a pattern segment is empty (:224-225), which breaks `subMap` bounds. The segment tree is correct. `TreeSetLookup` is the reference implementation, with no production user. +- **Trace:** `put(("",""),e)`; `get(pattern(""," "))` → [e]; the tree gives []. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testTreeSetLookupEmptySegmentPattern` (`@Disabled`). + +#### TRIE-11 — `initializeFrom` aliases the source's mutable tree + +- **Severity / status:** low / CONFIRMED (promoted: originating verdict PLAUSIBLE, 3/3 upheld). **Verification:** phase A 3/3 (trie-tla). **Origin:** trie-tla F9 (PLAUSIBLE), trie-lean observation (PLAUSIBLE). +- **Locations:** QNSTL:645, QNSTL:646. +- **Claim:** The root reference is copied, so a later put or clear mutates both lookups while each keeps its own, now stale, size. The Javadoc says "shallow copy, values are shared", so whether this is a bug depends on the contract. No caller in this repo. +- **Trace:** TLC `op_copy`: `B.initializeFrom(A)` with A empty, then `A.put(a,v1)` → B's tree holds 1 mapping while `B.size` is 0. +- **Models:** `formal/trie/tla/NOTES.md`, `formal/trie/lean/NOTES.md`. +- **Test:** proposed (TRIE-PR11, option 1 only). None until the contract is decided, because under option 2 the behaviour is correct. + +#### TRIE-12 — Glob patterns inherit the `"!"` upper bound; a wildcard-free glob gets range [`""`,`"!"`) and matches nothing + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean B10), 3/3 (ro-selfmatch #1), 3/3 (trie-tla F1). **Origin:** trie-lean B10 (part), trie-tla F1 (glob side-claim), ro-selfmatch #1. +- **Locations:** QNP:315, QNP:332, QNP:357, QNP:359, QNP:363. +- **Claim:** Three parts, fixed in three steps. (a) The glob upper-bound branch uses `"!"` (:357-359), so `createFromGlobs("*")` finds nothing. (b) The glob segment suffix is `'!'` (:363), as in TRIE-3. (c) `firstWildcardSeg` is initialised to 0 (:315), so the `!= -1` guard (:332) is always true and a wildcard-free glob gets [`""`,`"!"`). Low because `createFromGlobs` has no production caller (see the cluster note). +- **Trace:** tree {abc, foo.bar}: `createFromGlobs("abc")` → [] although it matches; `createFromGlobs("ab*")` → [abc]. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** part (a): the `createFromGlobs("*")` assertion in `QNLFT#testGlobLookupsFindMatches` (`@Disabled`). Parts (b) and (c): proposed. TRIE-PR3 updates the glob `"!"` expectations, and TRIE-PR9 adds `createFromGlobs("abc")` and `("foo","bar")` found with an exact `lowerInclusive()`. The wildcard-free case previously had only an ro-selfmatch scratch driver. + +#### TRIE-13 — Globs ending in `*` match deeper names, but the tree walks only one level + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean B10). **Origin:** trie-lean B10 (part). +- **Locations:** QNP:156. +- **Claim:** `isRecursivePattern()` is false for globs, so the tree walks only the level of the lower bound. `("a*","b")` gives [] in the tree and [ab] in `TreeSetLookup`. Low: glob API, no production caller. +- **Trace:** `put(a.x)`; `get(createFromGlobs("a*"))` → []. +- **Models:** `formal/trie/lean/NOTES.md`. +- **Test:** the `"a*"` assertion in `QNLFT#testGlobLookupsFindMatches` (`@Disabled`). + +#### TRIE-14 — Glob regexps are case-insensitive while the lookup bounds are case-sensitive + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean B10), 3/3 (trie-tla F11). **Origin:** trie-lean B10 (part), trie-tla F11. +- **Locations:** QNP:101, QNP:367, `com.avaloq.tools.ddk/src/com/avaloq/tools/ddk/caching/Regexps.java:54`. +- **Claim:** `Regexps.fromGlob(glob)` defaults to `ignoreCase=true` while the bounds use case-sensitive order. A glob `"F*"` matches `foo`, but the lookup misses it. Low: glob API, no production caller. +- **Trace:** `put(foo)`; `get(createFromGlobs("F*"))` → []. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** the `"F*"` assertion in `QNLFT#testGlobLookupsFindMatches` (`@Disabled`). + +#### TRIE-15 — Glob `matches()` throws `StringIndexOutOfBoundsException` on an empty last segment + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (trie-lean B10), 3/3 (trie-tla F12). **Origin:** trie-lean B10 (part), trie-tla F12. +- **Locations:** QNP:268. +- **Claim:** `lastSeg.charAt(lastSeg.length()-1)` throws when the last glob segment is empty. Low: glob API, no production caller. +- **Trace:** `createFromGlobs("a","").matches(("a",""))` → SIOOBE. +- **Models:** `formal/trie/lean/NOTES.md`, `formal/trie/tla/NOTES.md`. +- **Test:** `QNLFT#testGlobWithEmptyLastSegment` (`@Disabled`). + +#### TRIE-16 — `put(QualifiedName.EMPTY, v)` increments size and then throws + +- **Severity / status:** low / CONFIRMED (promoted: originating verdict PLAUSIBLE, 3/3 upheld). **Verification:** phase B 3/3 upheld, 2 overstated; suggested severity low ×3. **Origin:** trie-lean observation (PLAUSIBLE, not modelled). +- **Locations:** QNSTL:590. +- **Claim:** `size++` runs before `merge`, and `merge` calls `getSegment(0)`, which throws for `EMPTY`. +- **Narrowed (skeptics):** The sole production caller, PAEDL, discards the half-built lookup, so the inflated count is never observed. The practical issue is the exception type (AIOOBE instead of IAE). +- **Models:** `formal/trie/lean/NOTES.md` ("Observations"). +- **Test:** proposed (TRIE-PR6 case e). + +#### TRIE fix plans + +Single plan, no judge. **Version bumps (all TRIE PRs):** none right now: `com.avaloq.tools.ddk.xtext` is 17.4.1 against baseline v19.2.0 = 17.4.0 (bumped in d3b6e083a), and `com.avaloq.tools.ddk.xtext.test` is 17.3.2 against 17.3.1. If a release is published before a merge, micro-bump both (MANIFEST + pom). `Regexps` in `com.avaloq.tools.ddk` is called, not changed. No TRIE defect is shared with upstream Xtext; all affected classes are DDK-only. + +Common test rules: ship each fix and its test in one commit (history is rebase-merged). Port the `QNLFT` methods into the already-registered `QualifiedNamePatternTest` and `QualifiedNameSegmentTreeLookupTest`, removing them from `QNLFT` as they are ported. Show red on master and green with the fix under the aggregator, not only the javac harness. Strip `formal/` references from ported test comments. + +**TRIE-PR1 (TRIE-1)** + +- **Fix:** PAEDL:63 becomes `.matches(input.getName())`; `name` stays the caller's original-case pattern. +- **Files:** PAEDL, `PAEDLT` (enable). +- **Test strategy:** Enable the two `@Disabled` `PAEDLT` methods; the other 3 are guards. +- **Risks:** Visible behaviour change: case-sensitive `ContainerQuery` executions with a name pattern have returned [] since 4e2c12841 and now return results. Downstream code may have compensated, so a downstream smoke run is worthwhile. Single-element lookups are unaffected. A top-level `*` still returns nothing until TRIE-PR2. +- **Model evidence:** Lean `All.consumer_cs_empty`; ladder P4 fails until the "+F8 consumer" rung. TLA `co_cs` and `co_cs_fb` violate ConsCS (needed independently of the bounds); `co_fixed` passes; `co_wit` is reachable. + +**TRIE-PR2 (TRIE-2, TRIE-12)** + +- **Fix:** At QNP:374 (TRIE-2) and :359 (TRIE-12 step a), replace `QualifiedName.create("!")` with a private constant `UNBOUNDED = QualifiedName.create(String.valueOf(Character.MAX_VALUE))`, commented as equal to the sentinel from `QNSTL.init()` (:520). +- **Files:** QNP, `QualifiedNamePatternTest.java`, `QualifiedNameSegmentTreeLookupTest.java`. +- **Test strategy:** Port the two top-level-wildcard tests and the `createFromGlobs("*")` assertion. Add a consumer assertion: `getExportedObjects(ECLASS, create("*"), true)` over {b} returns b. In the same commit update `QualifiedNamePatternTest.testAllPattern` and `testRegexpPatterns` (`createFromGlobs("*")`, `("?")`, `("?foo*bar*")`). +- **Risks:** Top-level `*` and empty-prefix scopes now enumerate the whole container: correct, but potentially costly on large indexes; say so in the PR body. Relies on every live tree, including deserialized ones, holding the U+FFFF sentinel. Names starting with U+FFFF stay excluded (TRIE-9). +- **Model evidence:** Lean `All.topStar_unsound`; "+F1 topUpper" rung. TLA `pb_rs` violates RangeSound; `tq_spec`/`co_ci` fail on store {a} with `*`; `co_ci_fb` passes; `tq_fixed`/`tq_fixed5` pass. + +**TRIE-PR3 (TRIE-3, TRIE-12)** + +- **Fix:** Append `'\u0000'` instead of `'!'` at QNP:363 (TRIE-12 step b), :379, :382 and :386 (TRIE-3). The empty-segment special case (:380-383) collapses into the general case. Optional helper `successor(QualifiedName)`. The `(char)(c+1)` successor is left for TRIE-9. +- **Test strategy:** Port the two low-char tests. Add an ignore-case consumer assertion: `create("a")` over {a, "a "} returns only a. Update the `"foo!"` → `"foo\u0000"` expectations in `testQualifiedPrefixNamePattern`, `testRecursiveWildcardPattern`, `testPatternWithoutWildcard` and `testRegexpPatterns`. +- **Risks:** Result sets only shrink, and only for names with chars below `'!'`. Downstream code comparing against a literal `"!"` would break. Touches the same method as TRIE-PR2: merge after it, or combine. +- **Model evidence:** Lean `All.bang_not_successor`, `All.nul_is_successor`, "+F2 nulSucc". TLA `tq_spec`/`tq_star`/`tq_ref` fail; `tq_fixed`/`tq_fixed5` (35,443 stores) pass; `pb_rs2` shows the bug only over-includes. + +**TRIE-PR4 (TRIE-4)** + +- **Fix:** In the non-glob branch of `matches`, after the loop (:278-302): `return other.getSegmentCount() == getSegmentCount();`. +- **Test strategy:** In `QualifiedNamePatternTest`: `!create("a").matches(("a","b"))` and `create("a").matches(("a"))`; keep the tree-side consistency assertion. +- **Risks:** `ContainerBasedScope` (:86/:116) stops admitting longer names for programmatic wildcard-free pattern criteria, which then fall back to the parent scope. +- **Model evidence:** Lean `All.exact_matches_extensions`, "+F3 exactLen". TLA `pb_cnt1` violates CountFilterSound; `pb_fixed` passes on 714,096 pairs. + +**TRIE-PR5 (TRIE-10)** + +- **Fix:** Delete the empty-segment `return -1` (QNP:224-225). +- **Test strategy:** `testTreeSetLookupEmptySegmentPattern`, plus `("","b")` vs `("","a")`; extend `testComparison`. +- **Risks:** The public `Comparator` changes order only for empty-segment patterns, exactly where it was wrong. +- **Model evidence:** Lean "+F4 cmpEmpty"; P5 (subMap monotone) holds in every variant. TLA `tq2_ref` fails; `tq_fixed_ref`/`tq_fixed5` pass. + +**TRIE-PR6 (TRIE-5, TRIE-6, TRIE-16)** + +- **Fix:** (1) `ArrayUtils.addAll` dedupes against the growing result and routes the null/empty case through the same loop, returning `values` unchanged when it has no duplicates. (2) `putAll` passes a de-duplicated array to `merge`. (3) The private `merge` methods return the number of values added. (4) `put`/`putAll` do `size += root.merge(...)`, so a failed `put(EMPTY)` leaves size unchanged; an explicit `IllegalArgumentException` for empty names is optional. This replaces `fix-plan.patch` F5, which walks the tree twice and still counts `putAll(n,[v,v])` as 2. The value-sharing identity check (:436/:442) must compare against the de-duplicated array. +- **Files:** `ArrayUtils.java`, QNSTL, `QualifiedNameSegmentTreeLookupTest.java`. +- **Test strategy:** (a) put twice → 1 entry, remove → 0 (TRIE-5). (b) `putAll(a,[v,v])`, remove → unmapped, 0 entries (TRIE-6). (c) `put(a,w)`; `putAll(a,[v,v])` → [w,v], 2 entries. (d) (b) against `TreeSetLookup`. (e) `put(EMPTY)` throws and leaves 0 entries (TRIE-16). (f) After put/put/remove, `initializeFrom` succeeds. Run with `shareValues` both ways. +- **Risks:** `ArrayUtils` is a public util: `addAll(null|[], values)` now returns a new array when `values` has duplicates; the Javadoc identity promise still holds. `getEntries()` drops for re-puts. Trees deserialized from older caches keep the old size until rebuilt. +- **Model evidence:** TLA `op_sizetree`, `op_sizespec`, `op_nodup`, `op_exact` fail; `op_fixed`, `op_fixed5`, `op_fixed_sh`, `op_fixed_sh5` pass (TrieOps.tla:79/:88 model the length delta); `op_exactref` shows `TreeSetLookup` is also affected. Lean `All.addAll_present`, `All.put_size`, "+F5 size" (245,411 states). Lean does not cover TRIE-6; TRIE-16 is not modelled. + +**TRIE-PR7 (TRIE-7)** + +- **Fix:** Drop the `isBlank()` guard at :295. The outer `getMappings` iterates `root.children`, so the root's `""` is never prefixed. +- **Test strategy:** `testGetMappingsKeepsBlankSegments` for `(" ","a")` and `("","a")`, `shareValues` both ways; keep `testGetMappings` green. +- **Risks:** None beyond the correction. +- **Model evidence:** Lean "+F6 mappings". TLA `op_map` fails; `op_fixed`/`op_fixed_sh` pass. + +**TRIE-PR8 (TRIE-8)** + +- **Fix:** In `ValueSharingSegmentNode.matches` (:404-422), collect into a list when `excludeDuplicates` is false (`fix-plan.patch` F7), or delegate to `super.matches` in that case. +- **Test strategy:** `testValueSharingKeepsMultiplicity` → [v,v], plus a differential check against `shareValues=false` and `TreeSetLookup`. The `excludeDuplicates=true` result is unchanged. +- **Risks:** Negligible extra iteration; no in-repo caller. +- **Model evidence:** Lean P2 passes from "+F7 share" (20,876 stores × 315 patterns). TLA `op_share` fails; `op_bagns` passes as written; `op_fixed_sh`/`op_fixed_sh5` pass. + +**TRIE-PR9 (TRIE-12, TRIE-13, TRIE-14, TRIE-15)** + +- **Fix:** Lands after TRIE-PR2/3. (1) TRIE-12 step c: `firstWildcardSeg = -1` (:315); the else branch returns the exact plain name. (2) TRIE-13: `isRecursivePattern()` is true for globs. (3) TRIE-14: `Regexps.fromGlob(from, false)`; (3a) `toLowerCase()`/`toUpperCase()` (:188-190, :203-205) recompile the glob regexps from the case-mapped segments (missing from `fix-plan.patch`). (4) TRIE-15: :268 uses `endsWith` instead of `charAt(length-1)`. Alternative, given no production caller: deprecate `createFromGlobs` and ship only (4). +- **Test strategy:** One failing-first test per finding: `"a*"` over a.x, and `("a*","b")` tree == `TreeSetLookup` (TRIE-13); `"F*"` does not match foo but its `toLowerCase()` does (TRIE-14); `createFromGlobs("abc")` and `("foo","bar")` are found, with an exact `lowerInclusive()` (TRIE-12); an empty last segment does not throw (TRIE-15). +- **Risks:** Public API: glob case semantics and `isRecursivePattern()` change. Results remain candidate supersets. +- **Model evidence:** Lean P2g fails until "+F9 glob" and "+F10 globCase"; P1g holds under A1. TLA `pbg_rs` fails; `pbg_rs_fb` passes; `pbg_rs_fbA` fails (the case fix is needed); `pbg_exc` violates NoException; `pbg_fixed` passes on 874,104 pairs. **Caveat:** neither model covers (3a). + +**TRIE-PR10 (TRIE-9): issue first** + +- **Status:** deliberately left open by `fix-plan.patch` (assumption A1: no U+FFFF in names) until a design is chosen. +- **Fix options:** (a) a carry-aware prefix successor at :367/:389 that strips trailing U+FFFF and falls back to the parent's successor or `UNBOUNDED`; (b) an explicit unbounded upper bound plus a sentinel that name lookups cannot reach (structural; needs a serialization compatibility check); (c) document U+FFFF as reserved in the `QualifiedNameLookup`/`QNSTL` Javadoc. Recommended: (a) + (c), deferring (b). +- **Files:** QNP, QNSTL, `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/naming/QualifiedNameLookup.java`, `QualifiedNamePatternTest.java`. +- **Test strategy:** For (a): `TreeSetLookup.get(pattern("￿*"))` does not throw, and `pattern("a￿*").upperExclusive()` is `("b")`. `testMaxCharPattern`'s sentinel half stays `@Disabled` unless (b) is chosen. +- **Model evidence:** Lean P1b/P2m pass only under A1, so Lean gives no evidence for a U+FFFF fix. TLA `pb_ord` fails; `pb_fixed` passes with the carry-aware `PrefixLevelSucc`/`StripMax` and INF mapped to the sentinel. The trie-level collision is not covered. + +**TRIE-PR11 (TRIE-11): issue first** + +- **Fix options:** (1) a recursive structural copy that shares the value arrays (safe, because `ArrayUtils` is copy-on-write) and copies size; (2) keep the aliasing and document that the source must not be modified. Check downstream callers before choosing. +- **Test strategy:** Option 1: after `B.initializeFrom(A)`, neither `A.put` nor `A.clear()` affects B. Depends on TRIE-PR6. +- **Risks:** Option 1 makes `initializeFrom` O(n) instead of O(1). +- **Model evidence:** TLA `op_copy` violates CopySize; `op_fixed`/`op_fixed_sh` pass with FIX_COPY. Not modelled in Lean. + +### 3.5 PIPE — release pipeline (`.github/`) + +No upstream branch `v[0-9]*` exists today, so the maintenance-line findings (PIPE-1, PIPE-2, PIPE-4) become reachable when the first maintenance branch is cut. They stay high because that is a routine maintainer action, not a code change. + +#### PIPE-1 — Maintenance-branch builds always fail the Tycho baseline gate + +- **Severity / status:** high / CONFIRMED. **Verification:** phase A 3/3 (pipeline-lean), 3/3 (pipeline-tla). **Origin:** pipeline-lean F4, pipeline-tla F1 (part). +- **Locations:** `ddk-parent/pom.xml:64`, `ddk-parent/pom.xml:247`, `.github/scripts/publish-p2-ghpages.sh:69`, `.github/workflows/snapshot.yml:67`, `.github/workflows/verify.yml:79`, `.github/workflows/release.yml:74`. +- **Claim:** `baseline.repo.url` is `p2/releases/latest`, the highest release overall, and `compare-version-with-baselines` runs on every build (pom.xml:247-263). After master releases a higher version, a `v[0-9]*` branch fails with "Version has moved backwards" (`onIllegalVersion` defaults to fail in tycho-p2-extras 5.0.4). Maintenance snapshot, PR and release builds all fail. Independent of PIPE-2. +- **Trace:** push v18.x → snapshot 18.0.1 against baseline v19.0.0 → gate fails. W5/W2 are unreachable in all 254k states. +- **Models:** `formal/pipeline/lean/NOTES.md`, `formal/pipeline/tla/NOTES.md`, `formal/pipeline/tla/logs/asis_live_maint.log`. +- **Test:** proposed (PIPE-PR4): offline `test-resolve-baseline.sh`. + +#### PIPE-2 — Next release version comes from the highest tag in the whole repo + +- **Severity / status:** high / CONFIRMED. **Verification:** phase A 3/3 (pipeline-lean), 3/3 (pipeline-tla). **Origin:** pipeline-lean F3, pipeline-tla F1 (part). +- **Fix PR:** [#1550](https://github.com/dsldevkit/dsl-devkit/pull/1550) (merged) — `git tag --list 'v*' --merged HEAD`. Necessary but not sufficient for maintenance releases: PIPE-1 still blocks them. +- **Locations:** `.github/workflows/release.yml:44`, `:36`, `:52`, `:87`. +- **Claim:** release.yml:44 takes the highest `v*` tag overall. A maintenance release computes v19.0.(n+1) instead of v18.0.1, and the `feature.xml` check (:87-104) then always fails. Conversely, a master patch can pick up a maintenance tag. Fix: `--merged HEAD`. +- **Trace:** master releases v3.0; a maint/patch dispatch computes v3.1 instead of v2.1. +- **Models:** `formal/pipeline/lean/NOTES.md`, `formal/pipeline/tla/NOTES.md`, `formal/pipeline/tla/logs/asis_MaintOnOwnLine.log`, `formal/pipeline/tla/logs/asis_OwnLineVersion.log`. +- **Test:** proposed (PIPE-PR1): offline `test-next-release-version.sh`. + +#### PIPE-3 — Release tag pushed before the release repository exists; can be left permanently orphaned + +- **Severity / status:** high / CONFIRMED. **Verification:** phase A 3/3 (pipeline-tla), 3/3 (pipeline-lean). **Origin:** pipeline-tla F3, pipeline-lean F1. +- **Locations:** `.github/workflows/release.yml:106`, `:114`, `:130`, `:142`; `.github/scripts/publish-p2-ghpages.sh:57`; `.github/scripts/cleanup-p2-snapshots.sh:15`. +- **Claim:** `create_tag` pushes the tag (:106-112) before the `publish` job writes `p2/releases/` and the GitHub release (:114-148). If publish fails, "Re-run failed jobs" recovers only while `p2/snapshots/` survives cleanup (KEEP=20); after that `cp` fails (publish-p2-ghpages.sh:57). "Re-run all jobs" or a new dispatch computes the next version and fails the `feature.xml` check. Only a manual repair helps. Moving the tag push last on its own creates a new stuck state, so a resume step is also required. +- **Trace:** compute v19.0.1, checks pass, tag pushed → publish fails → newer snapshots evict `` → nothing can publish v19.0.1 (R1). +- **Models:** `formal/pipeline/tla/NOTES.md`, `formal/pipeline/lean/NOTES.md`, `formal/pipeline/tla/logs/asis_TagHasRepoOrLive.log`, `formal/pipeline/tla/logs/asis_live_tags.log`, `formal/pipeline/lean/report.txt`. +- **Test:** proposed (PIPE-PR6): failure-injection `test-release-resume.sh`. + +#### PIPE-4 — `p2/snapshots/latest` can point at a maintenance or arbitrary dispatched branch build + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase A 3/3 (pipeline-tla), 3/3 (pipeline-lean). **Origin:** pipeline-tla F2, pipeline-lean F2. +- **Locations:** `.github/scripts/publish-p2-ghpages.sh:72`, `.github/workflows/snapshot.yml:11`, `:26`, `ddk-parent/pom.xml:65`, `ddk-parent/pom.xml:387`. +- **Claim:** `snapshot.yml` runs for `v[0-9]*` pushes and for dispatches of any branch, and publish-p2-ghpages.sh:72 always rewrites `snapshots/latest`. `snapshot.repo.url` is also a tycho-p2-plugin `baselineRepositories` entry (pom.xml:387-393); that part is PLAUSIBLE in pipeline-lean and not modelled. +- **Trace:** push to maint → gate passes (no higher release yet) → S4 sets latest to the maint commit. +- **Models:** `formal/pipeline/tla/NOTES.md`, `formal/pipeline/lean/NOTES.md`, `formal/pipeline/tla/logs/asis_LatestFromMaster.log`. +- **Test:** proposed (PIPE-PR3). + +#### PIPE-5 — Shared concurrency group silently cancels pending runs across refs and workflows + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase A 3/3 (pipeline-tla), 3/3 (pipeline-lean). **Origin:** pipeline-tla F4, pipeline-lean F5. +- **Locations:** `.github/workflows/snapshot.yml:13`, `.github/workflows/release.yml:21`, `.github/workflows/release.yml:81`. +- **Claim:** Both workflows share the group `publish-ghpages-`. `cancel-in-progress: false` protects only the running run; GitHub replaces the pending one. A pending master snapshot can be cancelled by any maintenance push or dispatch, and release.yml:81-84 then refuses to release master HEAD. A pending release dispatch can be dropped. Relies on GitHub's documented semantics. +- **Trace:** release R1 running → master push (S pending) → R2 dispatch replaces S → R1 and R2 fail: HEAD has no snapshot. +- **Models:** `formal/pipeline/tla/NOTES.md`, `formal/pipeline/lean/NOTES.md`, `formal/pipeline/tla/logs/asis_NoCrossRefCancel.log`. +- **Test:** proposed (PIPE-PR7). + +#### PIPE-6 — A publish-job re-run cannot succeed once `gh release create` has run + +- **Severity / status:** low / CONFIRMED (promoted: originating verdict PLAUSIBLE, 3/3 upheld). **Verification:** phase A 3/3 (pipeline-tla), 3/3 (pipeline-lean). **Origin:** pipeline-tla F5 (PLAUSIBLE), pipeline-lean #5 part (PLAUSIBLE). +- **Locations:** `.github/workflows/release.yml:145`, `:152`. +- **Claim:** `gh release create` is not idempotent. If the job fails after the release exists (in the Summary step, or through a lost runner), every re-run fails with "already exists". Nothing is lost, but the run never turns green. That `gh` rejects an existing release was not checked offline (why the origin said PLAUSIBLE). +- **Trace:** T1-T4, P1-P3 (release created) → fail at P4 → re-run: P3 fails with "already exists". +- **Models:** `formal/pipeline/tla/NOTES.md`, `formal/pipeline/lean/NOTES.md`, `formal/pipeline/tla/logs/asis_RerunProgress.log`. +- **Test:** proposed (PIPE-PR5): stub `gh` on PATH. + +#### PIPE-7 — An 8-character short SHA computed in different clones may disagree + +- **Severity / status:** low / CONFIRMED (promoted: originating verdict PLAUSIBLE, 3/3 upheld). **Verification:** phase A 3/3 (pipeline-lean). **Origin:** pipeline-lean #5 part (PLAUSIBLE, not modelled). +- **Locations:** `.github/workflows/release.yml:63`, `.github/workflows/snapshot.yml:34`. +- **Claim:** Both sides use `git rev-parse --short=8` in different clones. Git lengthens an ambiguous prefix, so the release could fail to find its snapshot. +- **Models:** `formal/pipeline/lean/NOTES.md` (not modelled). +- **Test:** proposed (PIPE-PR2): static check plus `.commit` checks. + +#### PIPE fix plans + +Single plan, no judge. No PIPE PR touches bundle sources, so no MANIFEST/pom bumps. No PIPE defect is shared upstream. New tests live under `.github/scripts/test/` and run from a cheap new `verify.yml` job (no Maven), added by PIPE-PR1. + +**PIPE-PR1 (PIPE-2)** + +- **Fix:** Extract release.yml:44-57 into `.github/scripts/next-release-version.sh `, using `git tag --list 'v*' --merged HEAD --sort=-version:refname | head -1`. Guard: if `NEW_TAG` already exists, fail with `::error::`. +- **Files:** `.github/workflows/release.yml`, `.github/scripts/next-release-version.sh` (new), `.github/scripts/test/test-next-release-version.sh` (new), `.github/workflows/verify.yml`. +- **Test strategy:** Offline in a throwaway repo, all failing first: on v19.1.x a patch release gives v19.1.1; master ignores higher maintenance tags; a minor bump on v19.1.x that collides with an existing tag exits non-zero. +- **Risks:** A maintenance branch merged back into master does not change the result. Needs `fetch-depth: 0` (set, release.yml:39). Not sufficient without PIPE-1. +- **Model evidence:** TLA `fix_MaintOnOwnLine` holds (1.86M states). Lean `fixC_only` flips P3 only; `fixCD` makes W2 reachable. + +**PIPE-PR2 (PIPE-7)** + +- **Fix:** `git rev-parse HEAD | cut -c1-8` at snapshot.yml:34 and release.yml:63. Write the full SHA to `p2/snapshots//.commit`. The release check (release.yml:81) compares `.commit` with HEAD and warns when it is missing. +- **Files:** `snapshot.yml`, `release.yml`, `publish-p2-ghpages.sh`, `.github/scripts/test/test-sha8.sh` (new). +- **Test strategy:** Forcing a real ambiguity is impractical (~2^32 attempts). Instead: a static check for no `--short`; `.commit` present after publish; a mismatching `.commit` fails the release check. +- **Risks:** None functional. Snapshots without `.commit` age out of KEEP=20. +- **Model evidence:** Not modelled. + +**PIPE-PR3 (PIPE-4)** + +- **Fix:** (a) Rewrite `latest` only when `SNAPSHOT_REF` (dispatch input or `github.ref_name`) is `master`. (b) Write `$TARGET/.ref`. (c) Cleanup protects the `latest` target and the newest snapshot of master and of each `v[0-9]*` ref; the model showed (c) is required. Optional: relabel "Latest master snapshot" in `index.html`. +- **Files:** `publish-p2-ghpages.sh`, `cleanup-p2-snapshots.sh` (add a `KEEP=${KEEP:-20}` seam), `snapshot.yml`, `.github/scripts/test/test-publish-snapshot-latest.sh`, `.github/scripts/test/test-cleanup-protects-latest.sh` (new). +- **Test strategy:** Against a local bare origin: publishing maintenance after master keeps `latest` on master; `KEEP+1` maintenance snapshots do not evict the `latest` target; the newest `v19.1.x` snapshot survives. +- **Risks:** Consumers following a maintenance line via `latest` lose that. Must land before PIPE-PR4, which unmasks the bug. +- **Model evidence:** TLA `asis_LatestFromMaster` (6 states). `fix_safety`/`fix_safety_maint` hold LatestFromMaster and LatestExists; the cleanup protection was added after TLC found LatestExists failing; `plant_cleanup` is caught. Lean fixes B + F give P2/P7. Lean's fixed config sets `featDispatch=false`, so feature dispatches are covered only by the ref check in (a). + +**PIPE-PR4 (PIPE-1)** + +- **Fix:** New `.github/scripts/resolve-baseline.sh`: walk the tags merged into HEAD, pick the first with `p2/releases/$TAG` on `origin/gh-pages`, print its URL; fail if none (a silent fallback would make the gate pass vacuously). `snapshot.yml` and `verify.yml` pass `-Dbaseline.repo.url`. `pom.xml:64` keeps the default for local builds. On master this resolves to v19.2.0, as today. +- **Files:** `resolve-baseline.sh` (new), `snapshot.yml`, `verify.yml`, `.github/scripts/test/test-resolve-baseline.sh` (new), README.md or AGENTS.md (one line on local maintenance builds). +- **Test strategy:** Offline, failing first: resolves to v19.1.0 on the branch and v19.2.0 on master; a tag without a repo is skipped; no candidate exits non-zero. Manual Maven reproduction of "Version has moved backwards". A fork rehearsal writes to the fork's gh-pages and needs approval first. +- **Risks:** `verify.yml` exercises only the master path. Correctness relies on the tag-implies-repo order from PIPE-3; skipping repo-less tags keeps today's behaviour until then. Not sufficient without PIPE-2. +- **Model evidence:** The TLA fixed Gate uses the highest ancestor release; `asis_live_maint` violated, `fix_live_maint` holds. Lean `fixD_only` reaches W5 but not W2; `fixCD` reaches both. Gate semantics from the tycho-p2-extras 5.0.4 bytecode. + +**PIPE-PR5 (PIPE-6)** + +- **Fix:** (a) Zip from `p2/releases/$RELEASE_VERSION`, not from the snapshot. (b) `gh release view` → `upload --clobber`, else `create --verify-tag`. (c) In release mode the publish script writes `.commit`: same commit → no-op, different commit → fail. A published release repo is never overwritten. +- **Files:** `release.yml`, `publish-p2-ghpages.sh`, `.github/scripts/test/test-release-rerun.sh` (new). +- **Test strategy:** A stub `gh` on PATH. A second run after a simulated Summary failure exits 0 with one release and one asset; it still succeeds after the snapshot is deleted; a same-SHA republish is a no-op, a different SHA fails. +- **Risks:** Rests on `gh` rejecting duplicates (unchecked offline); the view-then-create form is correct either way. Prerequisite for PIPE-PR6's resume step. +- **Model evidence:** TLA `asis_RerunProgress` violated; the fixed P3 holds RerunProgress. Lean fix A makes W3 reachable. + +**PIPE-PR6 (PIPE-3)** + +- **Fix:** One job: prepare → publish the repo → push the tag → create the GitHub release, each step idempotent. (1) Drop the `branch` input and pin `github.sha`; dispatch via "Use workflow from". (2) Resume first: a tag at HEAD, or an orphan `p2/releases/` above the highest merged tag whose `.commit` is an ancestor; otherwise compute the version (PIPE-2). (3) The snapshot check also accepts a matching release repo. (4)-(7) `feature.xml` check, publish, tag push (skip if already at this commit, fail if elsewhere), GitHub release (PIPE-6). +- **Files:** `release.yml`, `.github/scripts/release.sh` (new), `publish-p2-ghpages.sh`, `.github/scripts/test/test-release-resume.sh` (new), README.md (dispatch instructions). +- **Test strategy:** Failure injection in the harness only: a pre-receive hook rejects tags once; the stub `gh` fails create once. Scenario A (orphan tag) is red on today's order and resumes after the fix. B: a new commit between attempts; the pinned re-run finishes ``. C: re-running a completed release is a no-op. A fork rehearsal needs approval. +- **Risks:** Dispatch changes for maintainers. The `environment: release` branch rules (not visible offline) must allow `v[0-9]*`. Cherry-pick onto any maintenance branch cut earlier. Repos published before this change have no `.commit`. There is a short window where `releases/latest` moves before the tag exists. Largest PR: review as an extraction commit plus a behaviour commit. +- **Model evidence:** TLA `asis_TagHasRepoOrLive` and `asis_live_tags` violated. The fixed ordering plus resume plus pinned SHA hold TagHasRepoOrLive, RerunProgress and NoLostContent (`fix_safety_wide`, 177M states) and LTags/LGoal; resume and pin were each added after TLC counterexamples on earlier fixed models. Lean: faithful R1 fails (10-step stuck trace); tag-last without resume breaks R2, so resume is mandatory. + +**PIPE-PR7 (PIPE-5)** + +- **Fix:** (1) Per-workflow, per-ref groups (`snapshot--`, `release--`), keeping `cancel-in-progress: false`. (2) Both gh-pages scripts wrap their mutation in `apply`; on a rejected push they fetch, reset, re-apply and retry (bounded, with jitter) instead of `git rebase`. Never force-push. +- **Files:** `snapshot.yml`, `release.yml`, `publish-p2-ghpages.sh`, `cleanup-p2-snapshots.sh`, `.github/scripts/test/test-ghpages-concurrent-writers.sh` (new). +- **Test strategy:** A static YAML check of the group keys. A stale-checkout race test: today the rebase conflicts on `index.html`; after the fix both snapshots exist, `releases/latest` is the maximum, nothing is lost. Also the reverse cleanup interleaving. +- **Risks:** Removes the global serialisation that makes several invariants hold trivially; correctness then rests on regenerate-on-retry. Needs PIPE-PR3 and PIPE-PR6 first. Model evidence at larger bounds is thinner. Interim alternative: keep one writer group and add only a separate release group. +- **Model evidence:** TLA `asis_NoCrossRefCancel` (4 states) violated; with `<>` groups and atomic recompute, `fix_safety`, `fix_safety_maint` and `fix_safety_wide` hold everything. Lean `fixes_without_E` fails only P5; the full config holds all 15 verdicts at budget 1, keep 1/2; perBranch at budget 2 not run (>5M states); `planted_caught` (force push) violates P4. + +### 3.6 RO — other code-read findings + +**Severity note:** RO-2, RO-4, RO-5, RO-6, RO-7 and RO-8 concern test code and CI only, with no shipped-code impact, so all are low. RO-2 is the only one with an effect today, since it leaves 4 test methods unrun since 2017. RO-4 is masked by RO-2 until the bundle is wired. RO-7 is the root cause that let RO-2 persist; it is sequenced last for mechanical reasons (it turns master red until the others land), not because of severity. RO-1 is medium: interleaved locators are the normal case and corrupt the formatter's column stacks. RO-3 needs 32 or more locators on one token boundary, which is not realistic for normal formatter configurations, so the crash is low; its fix is RO-1's. + +#### RO-1 — `sortLocators` comparator is not a total order + +- **Severity / status:** medium / CONFIRMED. **Verification:** phase A 3/3 (ro-comparator). **Origin:** ro-comparator. +- **Locations:** EFCBS:547, EFCBS:559, EFCBS:322, EFCBS:569. +- **Claim:** The comparator returns 0 whenever either argument is not a `FixedLocator`, so equality is not transitive. For lists under 32 elements TimSort finds one run and changes nothing, so interleaved `FixedLocator`s stay unsorted. `processColumnLocators` relies on "openers first, then by column". With a closer first, `lastIndexOf` returns -1, an unrelated pair is popped, and the `columnIndents`/`initialIndents` stacks are corrupted. Interleaving is normal (the `activeRangeLocators` `HashSet` is mixed in). Wrong end-to-end formatter output was not shown. +- **Trace:** `[open@5, NoFormatLocator, open@1]` → unchanged; `[close@5, NoFormatLocator, open@5]` → unchanged; a random probe left 15,028 of 20,000 lists unsorted. +- **Models:** `formal/readonly/comparator/Runner.java`, `formal/readonly/comparator/MinSize.java`. +- **Test:** `SLT#testFixedLocatorsAreSortedByColumnAcrossOtherLocator` and `#testOpeningFixedLocatorPrecedesClosingAcrossOtherLocator` (`@Disabled`, plus reflective Runner); `#testFixedLocatorsAreSortedByColumn` is a control that passes and runs. + +#### RO-2 — `com.avaloq.tools.ddk.test.ui.test` is built but its tests never run + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (ro-orphan). **Origin:** ro-orphan. +- **Locations:** `ddk-parent/pom.xml:44`, `:105`, `:270`; `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/AllTests.java:38`; `com.avaloq.tools.ddk.xtext.test/META-INF/MANIFEST.MF:9`; `com.avaloq.tools.ddk.test.ui.test/META-INF/MANIFEST.MF:1`; `com.avaloq.tools.ddk.test.ui.test/pom.xml:12`. +- **Claim:** `DeChKeyboardLayoutTest`, `SwtBotRadioTest` and `ErrorLogListenerTest` (4 `@Test` methods) are compiled but never run. The module inherits `skip=true`, nothing reachable from `com.avaloq.tools.ddk.xtext.AllTests` selects them, the aggregator does not import the bundle, and the bundle exports nothing. They have not run in CI since the bundle arrived (d08ea194a, 2017). **Severity rationale:** lowered from medium per the cluster note; the impact is lost test coverage, not shipped behaviour. +- **Trace:** `AllTests` selects 14 suites, none from `test.ui.test`; `git log -S'test.ui.test'` on the aggregator MANIFEST and `AllTests` is empty. +- **Models:** `formal/readonly/orphans/check-test-reachability.sh`, `formal/readonly/orphans/selftest.sh`, `formal/readonly/orphans/expected.txt`. +- **Test:** the reachability script exits 1 with UNREACHABLE=3 on a `git archive HEAD` snapshot, and exits 0 after wiring (mutation check); `selftest.sh` passes. Run it with `formal/check.sh --only=orphans`. + +#### RO-3 — The `sortLocators` comparator makes `Collections.sort` throw for 32 or more locators + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (ro-comparator). **Origin:** ro-comparator. +- **Locations:** EFCBS:548. +- **Claim:** At `MIN_MERGE`=32, TimSort can detect the contract violation and throw `IllegalArgumentException`, aborting formatting. It needs 32 or more locators on one token boundary (see the severity note). The RO-1 fix also fixes this. +- **Models:** `formal/readonly/comparator/MinSize.java`, `formal/readonly/comparator/Runner.java`. +- **Test:** No JUnit test; `SLT` uses lists of 2-3 locators. `MinSize.java` shows the first IAE at n=32, and the `Runner` probe saw 2,751 of 20,000 lists throw. Proposed (RO-PR1): a fixed-seed large-list case. + +#### RO-4 — `ErrorLogListenerTest` is missing even from its own bundle's `AllTests` + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (ro-orphan). **Origin:** ro-orphan. +- **Locations:** `com.avaloq.tools.ddk.test.ui.test/src/com/avaloq/tools/ddk/test/ui/test/AllTests.java:24`, `com.avaloq.tools.ddk.test.ui.test/src/com/avaloq/tools/ddk/test/ui/test/logging/ErrorLogListenerTest.java:29`. +- **Claim:** The bundle `AllTests` selects only the other two classes, so wiring the bundle alone would still leave this one orphaned. +- **Models:** `formal/readonly/orphans/check-test-reachability.sh`. +- **Test:** the reachability script lists it as unreachable (exit 1). + +#### RO-5 — `ErrorLogListenerTest` does not actually test ignoring, and its ignore location is stale + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (ro-orphan). **Origin:** ro-orphan. +- **Locations:** `ErrorLogListenerTest.java:66`, `com.avaloq.tools.ddk.test.core/src/com/avaloq/tools/ddk/test/core/util/ErrorLogListener.java:177`. +- **Claim:** The ignore location names the old package `...test.core.util.ErrorLogListenerTest`. The test then asserts `isExceptionLogged(NPE)`, which never consults the ignore lists (they are read only by `isException()`, `ErrorLogListener.java:138-184`), so it passes even if ignoring is broken. +- **Models:** code read only; the class's unreachability comes from `formal/readonly/orphans/check-test-reachability.sh`. +- **Test:** proposed (RO-PR3): wait for the exact NPE and assert by identity it is not logged, plus a control without the ignore; mutation checks (stale location, or `isExceptionIgnored` forced false) must fail it. + +#### RO-6 — `DeChKeyboardLayoutTest` uses a stale keyboard-layout name and leaks global SWTBot preferences + +- **Severity / status:** low / CONFIRMED (promoted: originating verdict PLAUSIBLE, 3/3 upheld). **Verification:** phase A 3/3 (ro-orphan). **Origin:** ro-orphan (PLAUSIBLE; inferred from javap, not executed). +- **Locations:** `com.avaloq.tools.ddk.test.ui.test/src/com/avaloq/tools/ddk/test/ui/test/swtbot/DeChKeyboardLayoutTest.java:34`, `com.avaloq.tools.ddk.test.ui/src/com/avaloq/tools/ddk/test/ui/swtbot/DE_CH.keyboard`, `com.avaloq.tools.ddk.test.ui/src/com/avaloq/tools/ddk/test/ui/swtbot/util/SwtBotUtil.java:32`. +- **Claim:** `KEYBOARD_LAYOUT="com.avaloq.test.swtbot.DE_CH"`, but the file moved in 27d54e7e8 (2017); SWTBot throws IAE for a missing layout. The test also never restores `KEYBOARD_LAYOUT`/`KEYBOARD_STRATEGY`. +- **Models:** code read only (javap of SWTBot's layout lookup); the class's unreachability comes from `formal/readonly/orphans/check-test-reachability.sh`. +- **Test:** proposed (RO-PR3): wire first and observe the IAE locally, then fix. + +#### RO-7 — The CI guard cannot detect orphaned tests + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (ro-orphan). **Origin:** ro-orphan. +- **Locations:** `.github/scripts/check-surefire-reports.sh:6`, `.github/workflows/verify.yml:82`. +- **Claim:** The guard fails only if no surefire report exists at all, so any unselected test class passes CI silently. That is why RO-2 went unnoticed. +- **Models:** `formal/readonly/orphans/check-test-reachability.sh`, `formal/readonly/orphans/selftest.sh`. +- **Test:** the replacement guard exists as `formal/readonly/orphans/check-test-reachability.sh` and exits 1 on unreachable tests; proposed (RO-PR5): move it into CI. + +#### RO-8 — The aggregator selects an empty placeholder suite (`CheckCfgUiTestSuite`) + +- **Severity / status:** low / CONFIRMED. **Verification:** phase A 3/3 (ro-orphan). **Origin:** ro-orphan. +- **Locations:** `com.avaloq.tools.ddk.checkcfg.ui.test/src/com/avaloq/tools/ddk/checkcfg/ui/test/CheckCfgUiTestSuite.java:18`, `com.avaloq.tools.ddk.xtext.test/src/com/avaloq/tools/ddk/xtext/AllTests.java:50`. +- **Claim:** It has no `@Suite`/`@SelectClasses`, and the bundle has no tests. It looks like coverage but runs nothing. +- **Models:** `formal/readonly/orphans/check-test-reachability.sh`. +- **Test:** the script reports INFO "selected-but-empty" and does not fail. + +#### RO fix plans + +Single plan, no judge. No RO defect is shared upstream. + +**RO-PR1 (RO-1, RO-3)** + +- **Fix:** Replace the comparator in `sortLocators` (EFCBS:547), keeping the private signature: non-`FixedLocator`s go last and compare equal to each other; `FixedLocator`s are ordered by `(getLeft() != null)`, then by column. Safe because only `processColumnLocators` reads the order and it skips non-Fixed entries, and `getLocators` returns a `HashSet` anyway (:325). For all-Fixed lists the result is identical to today's. Alternative: sort only the Fixed subsequence (requires rewriting the reflective test). Update the Javadoc. +- **Files:** EFCBS, `SLT` (enable). +- **Version bumps:** `com.avaloq.tools.ddk.xtext`: none at 17.4.1 against 17.4.0; if 17.4.1 is the published baseline by merge time, bump to 17.4.2. `xtext.test`: none. +- **Test strategy:** Enable the two bug cases, and add a fixed-seed (42) large-list case over 2-80 mixed locators asserting no throw and a sorted Fixed subsequence (RO-3). Regression coverage: `FormatterTest.column()` and `FormatTestSuite`. +- **Risks:** Formatter output can change only where the old stacks were corrupted; describe the PR as a correctness and crash fix. The Javadoc must state that nothing reads the list order after `processColumnLocators`. +- **Model evidence:** `Runner.java` and `MinSize.java`. Planned ablation: rerun them on the patched bundle and expect 3/3 PASS, unsorted=0, IAE=0; a half-fix that keeps `return 0` for mixed pairs must still fail. + +**RO-PR2 (optional: RO-A1, RO-A2)** + +- **Fix:** (RO-A1) `com.avaloq.tools.ddk.test.core/src/com/avaloq/tools/ddk/test/core/jupiter/IssueAwareRule.java:87`: restore `!issueAnnotation.fixed()` (the JUnit 4 original, d08ea194a, had the negation; the Jupiter port c0dd0ad4f dropped it). (RO-A2) `ErrorLogListener.LogListener`: a thread-safe list, with accessors returning snapshot copies. +- **Files:** `IssueAwareRule.java`, `ErrorLogListener.java`, `com.avaloq.tools.ddk.test.core/META-INF/MANIFEST.MF`, `com.avaloq.tools.ddk.test.core/pom.xml`, a new Mockito-based `IssueAwareRuleTest` in `xtext.test`, registered in `XtextTestSuite`. +- **Version bumps:** `com.avaloq.tools.ddk.test.core` 18.0.1 → 18.0.2 (required; it ships in the feature). +- **Test strategy:** Mock `ExtensionContext` and `Invocation`. With `fixed=false`, a throwing invocation passes and a passing one fails with "must fail"; with the default `fixed=true`, the invocation's result propagates. Red on master (RO-A1 confirmed in phase C). No deterministic test for RO-A2. +- **Risks:** Published test API: downstream tests using the default `fixed=true` will now run normally and may surface real failures. `SwtBotRadioTest`'s annotation must be set from an observed local run. +- **Model evidence:** None; git diff and the Javadoc example. RO-A2: `org.eclipse.core.jobs` 3.15.900 bytecode shows `RuntimeLog.log` running on the worker thread. + +**RO-PR3 (RO-2, RO-4, RO-5, RO-6)** + +- **Fix:** (1) `ErrorLogListenerTest` (RO-5): use `ErrorLogListenerTest.class.getName()` as the location; after `join()`, wait up to 10 s for the exact NPE (`JobManager.endJob` logs after releasing `join()`), assert by identity it is not in `getLoggedExceptions()`, and add a control case without the ignore. (2) `DeChKeyboardLayoutTest` (RO-6): layout `com.avaloq.tools.ddk.test.ui.swtbot.DE_CH`; save and restore the preferences and close editors in finally; fix the `SwtBotUtil.java:32` comment. (3) Add `ErrorLogListenerTest` to the bundle suite (RO-4). (4) Rename the bundle suite to `TestUiTestSuite` (and its `.launch`), export `com.avaloq.tools.ddk.test.ui.test`, import it in `xtext.test`, and select it last in `AllTests` (RO-2). +- **Files:** `ErrorLogListenerTest.java`, `DeChKeyboardLayoutTest.java`, `AllTests.java` → `TestUiTestSuite.java`, `AllTests.launch` → `TestUiTestSuite.launch`, `test.ui.test` MANIFEST, `SwtBotUtil.java` (comment), `xtext.test` MANIFEST, `xtext.test` `AllTests.java`, possibly `SwtBotRadioTest.java` (`@Issue` flag). +- **Version bumps:** `test.ui.test`: optional 17.3.1 → 17.3.2. `xtext.test`: none. `com.avaloq.tools.ddk.test.ui` (comment only): none at 17.3.4 against 17.3.3; if 17.3.4 is published by merge time, bump to 17.3.5 or drop the comment fix. +- **Test strategy:** Wire first, locally: `DeChKeyboardLayoutTest` should fail with IAE, and `ErrorLogListenerTest` passes regardless. Then fix. Mutation checks as under RO-5. Confirm with the reachability script and `surefire-reports` that all 4 methods ran. +- **Risks:** Wiring runs `SwtBotRadioTest` under `IssueAwareRule`, so land RO-PR2 first (RO-A1 is confirmed). The DeCh outcome after the rename is inferred, not run; if unstable, `@Disabled` it with a reason. Window > Preferences is missing on macOS, so local macOS runs may fail. Adds aggregator time. The rename breaks external launch configs. +- **Model evidence:** Reachability script output; `ErrorLogListener.java:138-184`; javap of `JobManager.endJob` (offset 19 before 72); the only `DE_CH.keyboard` is under `com.avaloq.tools.ddk.test.ui`. + +**RO-PR4 (RO-8)** + +- **Fix:** Remove `CheckCfgUiTestSuite` from `AllTests` and its import from the `xtext.test` MANIFEST. Do not annotate it with `@Suite` (`failIfNoTests=true` would break discovery). Optionally delete the placeholder bundle (module entry at `ddk-parent/pom.xml:122`). +- **Version bumps:** none. +- **Test strategy:** None possible. The script's INFO line disappears; surefire counts are unchanged. +- **Risks:** Minimal. + +**RO-PR5 (RO-7)** + +- **Fix:** Move the reachability script and its selftest to `.github/scripts/`, replacing `rg` with `git ls-files`/`find` and `grep -Eq`. Add an early `verify.yml` step that runs the selftest, then the check. Keep `check-surefire-reports.sh`. Optional strict mode that fails on selected-but-empty. +- **Files:** `.github/scripts/check-test-reachability.sh` (new), `.github/scripts/selftest-test-reachability.sh` (new), `.github/workflows/verify.yml`. +- **Version bumps:** none. +- **Test strategy:** The selftest's 3 synthetic cases. The real check exits 1 on current master and 0 after the other RO PRs. +- **Risks:** The regex/awk parser does not handle `@SelectPackages`, `@Nested` or pattern selection (none used today); document this. Must land last, or master CI goes red. + +## 4. Dropped and refuted items, and observations + +### 4.1 Refuted and plausible findings + +- **STO-7 (REFUTED, 0/3).** Reasons under [STO-7](#sto-7--swallowed-ioexception-in-writeresource-keeps-the-old-binary-while-the-uri-leaves-sources). No fix. +- **STO-6 (PLAUSIBLE, 1/3).** Reasons under [STO-6](#sto-6--after-a-link-exception-the-resource-is-detached-but-still-stored). The fix (STO-PR4) is kept, framed as "do not serialise a detached resource". + +### 4.2 Dropped items + +**Merged or split during cataloguing** + +- ro-selfmatch #0 (case-sensitive self-match in PAEDL): merged into TRIE-1. +- trie-tla F4 (the sibling loop visits a sibling's descendants but not the sibling; 2/3 upheld, one skeptic said the trace does not exercise the loop): same root cause as the `'!'` gap, unreachable once `'\0'` is used. Merged into TRIE-3. +- trie-tla F1 side-claim (`firstWildcardSeg` initialised to 0, dead `-1` guard): merged into TRIE-12 with ro-selfmatch #1. +- trie-lean B10 (a bundle of 4 glob problems): split into TRIE-12, TRIE-13, TRIE-14, TRIE-15. +- pipeline-tla F1 (a maintenance line cannot release): split into PIPE-1 and PIPE-2, matching pipeline-lean F4/F3. +- pipeline-lean #5: split into PIPE-7 and PIPE-6; the `gh release` part merged with pipeline-tla F5. +- storage round 2 (TLA B3 vs Lean B4+B5): re-split by defect into STO-4 and STO-5. +- REF judge graft 8 (upstream `batchedSearchResultEvents` not cleared on `setInput`): folded into REF-5 as its upstream variant. + +**Not defects of the as-written code** + +- Pipeline fixed-model constraints: cleanup must protect the `latest` target (in PIPE-PR3), re-runs must pin `github.sha`, and half-done releases must be resumable (both in PIPE-PR6). +- STO: the `RejectedExecutionException` catch at MCBS:727-734 is dead code (`CallerRunsPolicy` never discards). Harmless. +- STO Lean note "serialisation of a detached resource after a timeout": a model simplification, folded into STO-5. +- REF Lean false alarm (ghost `resetPending` cleared too early in 'lost+order+snap'): a model artefact, fixed in the model. +- REF "Reset does not clear `batchAddNodes`": never visible, because the same UI job's `refresh()` removes the nodes (P3 holds). Latent fragility; REF-PR5 clears it anyway. +- REF "`asyncExec` alone is not a fix for REF-1": a fix-design constraint (in REF-PR5). +- TRIE Lean: double-checked locking in PAEDL:95-98 re-tests a local that is always null. Benign: racing threads build equivalent lookups, last write wins. +- LDR "upstream Xtext `ParallelResourceLoader` has the same code": context for LDR-1. +- Trie properties checked and found sound: exact get, non-top wildcards without chars below `'!'`, the `find()` null path, the LastLoop result, and `remove`/`removeMappings` accounting. + +**Artefacts and environment** + +- ro-orphan #6 (campaign tests not registered in any suite): an artefact of the spike's working tree, not a repo defect. At the time of phase A, `QNLFT`, `PAEDLT` and `SLT` were untracked and unregistered, and `PRLT` and `FRSRCPT` were untracked and registered only by uncommitted suite edits. This PR commits all five and registers them in `XtextTestSuite`/`XtextUiTestSuite`, with their failing methods `@Disabled`. +- Local JDK-27 failures unrelated to the campaign (`ScopeExpressionCodeGenerationTest.testNestedArithmetic`, `IssueExpressionGenerationTest.testDynamicMarkerFeature`, `CheckApiAccessValidationsTest.testNonAvaloqTypeAccessable`) and the `CheckQuickfixTest.testBulkApplyingQuickfix` flake: pre-existing. + +### 4.3 Verified observations + +Raised by judges or planners, or by skeptics as out-of-scope side notes, then put through the three phase C skeptics. All 9 were upheld 3/3; most were narrowed. They are counted separately from the 51 catalogued findings (rule 3). + +| ID | Status | Sev | Observation | Location | Why it matters (as raised) | Narrowed (skeptics) | Source | Plan | +|---|---|---|---|---|---|---|---|---| +| REF-A1 | CONFIRMED (3/3, 3 overstated) | low | `dispose()` never calls `super.dispose()`, so `detachListenerFromIndex` never runs; the constructor registers a second index listener on top of super's `attachListenerToIndex`. The REF judge checked the code. | FRSRCP:104-106, FRSRCP:69-71 | Each disposed provider stays registered on the index (leak) and receives `descriptionsChanged` twice. | Downstream-only (DDK never binds FRSRCP). The leak is real: each disposed provider and its viewer stay on the index and get a no-op `asyncExec` per index change. No double notification: the listener set is a `CopyOnWriteArraySet`. | REF plan B, REF judge | REF-PR7 | +| REF-A2 | CONFIRMED (3/3, 0 overstated) | low | Upstream `remove()` uses super's private `rootNodes`, which DDK never populates, so `RemoveSelectedMatchesAction` NPEs. The REF judge checked the code; the NPE has not been reproduced. | Xtext `ReferenceSearchResultContentProvider:295`, `ReferenceSearchViewPageActions:150` | "Remove selected matches" in the search view would throw. | As stated, downstream-only: where FRSRCP is bound, "Remove Selected Matches" or DEL with a selection always NPEs (logged); nothing is removed, no data lost. Confirmed from bytecode, not run. | REF plan B, REF judge | REF-PR7 | +| STO-A1 | CONFIRMED (3/3, 3 overstated) | low | A storage worker's `PortableURIs.toPortableURI` reads the builder's `ResourceSetImpl` (`getEObject(uri, false)`) while the builder adds resources. | Xtext `PortableURIs:183`, reached from `DirectLinkingResourceStorageWritable:209` | Another cross-thread read on a non-thread-safe resource set, which could write non-portable references; not fixed by any STO PR. | Downstream-only, and only with DLRSF bound alongside stock Xtext `PortableURIs` (DDK's `DirectLinkingPortableURIs` never reads the resource set). A racy miss writes an `#UNRESOLVABLE` reference, not a non-portable one. | STO plan B, STO judge ("file as new finding") | none yet | +| STO-A2 | CONFIRMED (3/3, 3 overstated) | low | `deleteBinaryResources` dereferences `getResourceServiceProvider(uri)` without a null check. | MCBS:782 | A load can fail because no language handles the URI; once STO-PR4 calls `deleteBinaryResources` on the failure path, an NPE there would escape the catch and abort the build. | Downstream-only. `ToBeBuiltComputer.removeStorage` already drops provider-less URIs, so it needs stale persisted-index entries for a deregistered language; the NPE is caught by `update()`, rolls back and forces a full rebuild. | STO plan A, STO judge | STO-PR4 (optional) | +| STO-A3 | CONFIRMED (3/3, 3 overstated) | low | After a `StackOverflowError` inside linking, the half-linked but still attached resource is stored. | MCBS:578-581 | A binary of an incompletely linked resource may be written. The judge called skipping it "a separate behaviour choice, not a confirmed bug". | Downstream-only hygiene: the "no linking errors" gate is bypassed (an SOE adds no diagnostic), but unresolved references stay lazy-link proxies, so the stored model is not wrongly linked. | STO plan B, STO judge | none (reviewer note) | +| STO-A4 | CONFIRMED (3/3, 3 overstated) | low | `CheckBatchLinkableResourceStorageFacade`'s Xbase-based writable declares `throws IOException`, so Xtext RSF's swallowing catch could be reached there; the sink is an in-memory stream, so practically unreachable. | Xtext RSF:97-103 | The residue of refuted STO-7 for non-DLRSF facades; a stale binary would be kept if it ever fired. | Downstream-only and effectively unreachable: the only `IOException` source is a broken inferrer invariant (JvmTypes without a `JvmModelAssociator` adapter). The enabling removal is MCBS:656 (STO-1), not 754. | STO-7 skeptics | none | +| TRIE-A1 | CONFIRMED (3/3, 3 overstated) | none (not a defect) | `TreeSetLookup.getStatistics()` counts names (`lookupMap.size()`), while the segment tree counts name-value pairs. | `TreeSetLookup.java:127-128` | Monitoring-only inconsistency between the two lookups. The TRIE plan notes it and leaves it out of scope. | Accurate but diagnostic-only: `CacheStatistics.entries` has no defined meaning for multimaps, `TreeSetLookup` matches `MapCache`'s key count, is test-only and never registered with `CacheManager`. | TRIE plan | none | +| RO-A1 | CONFIRMED (3/3, 0 overstated) | medium | `IssueAwareRule` has an inverted `fixed()` check (regression from c0dd0ad4f): `@Issue(fixed=false)` tests run normally, and default-annotated tests are expected to fail. | `com.avaloq.tools.ddk.test.core/src/com/avaloq/tools/ddk/test/core/jupiter/IssueAwareRule.java:87` | Published test API with the opposite of its documented behaviour; affects `SwtBotRadioTest` once RO-2 is fixed. | As stated. Latent in DDK CI (`SwtBotRadioTest` is not aggregated, RO-2), but downstream `AbstractTest` subclasses with a default `@Issue` pass real `AssertionError`s silently. Medium per rule 2: silent wrong verdicts behind a downstream-only trigger. | RO plan (found while planning; formerly NEW-RO-A) | RO-PR2 (optional) | +| RO-A2 | CONFIRMED (3/3, 2 overstated) | low | `ErrorLogListener.LogListener` keeps statuses in a plain `ArrayList` that job worker threads write while the test thread reads. | `com.avaloq.tools.ddk.test.core/src/com/avaloq/tools/ddk/test/core/util/ErrorLogListener.java:206` | A data race in test infrastructure that can make log assertions flaky. | Real race (listeners run synchronously on the logging thread), downstream-only since `ErrorLogListenerTest` never runs. The larger flakiness source is ordering: `JobManager.endJob` logs after releasing `join()`, which a thread-safe list does not fix. | RO plan (found while planning; formerly NEW-RO-B) | RO-PR2 (optional) | + +## 5. Proposed fix-PR sequence + +Ordered by severity × effort, subject to the ordering constraints. Each PR is small and self-contained, branched from an explicit `upstream/master`, and opened as a draft. Before pushing, run `mvn -T 3C verify` plus `checkstyle:check`, `pmd:check` and `spotbugs:check`. Use `-step-N` branch names only if PRs are stacked. + +**Tests carried by this reference PR.** Each fix PR removes the `@Disabled` from its test (or ports it) in the same commit as the fix: + +- `PRLT` → LDR-PR1 (documents the first fix; [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) brings its own tests) +- `FRSRCPT` → REF-PR1 (done in [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552)) +- `PAEDLT` → TRIE-PR1 (done in [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551)) +- `QNLFT` methods → ported into `QualifiedNamePatternTest`/`QualifiedNameSegmentTreeLookupTest` by TRIE-PR1..PR9; `testMaxCharPattern` stays disabled until TRIE-PR10 is decided. Delete `QNLFT` once empty. +- `SLT` → RO-PR1 +- `formal/readonly/orphans/check-test-reachability.sh` → RO-PR5 (moved into `.github/scripts/`) + +**Shared version bumps.** `com.avaloq.tools.ddk.xtext.builder` 17.3.1 → 17.3.2: whichever of LDR-PR1/2/3 and STO-PR1..5 lands first. `com.avaloq.tools.ddk.xtext.ui` 17.3.3 → 17.3.4: the first REF PR. `com.avaloq.tools.ddk.test.core` 18.0.1 → 18.0.2: RO-PR2. TRIE, PIPE and the other RO PRs need no bump unless a release is published before they merge. + +| # | PR | Findings | Sev | Effort | Hard ordering constraints | +|---|---|---|---|---|---| +| 1 | TRIE-PR1 — merged as [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551) | TRIE-1 | high | one line + test | none | +| 2 | LDR-PR1 — opened as [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) (draft); a timeout abandons the load operation | LDR-1 | medium | small + tests | first builder bump | +| 3 | TRIE-PR2 | TRIE-2, TRIE-12 | high | small | none | +| 4 | PIPE-PR1 — merged as [#1550](https://github.com/dsldevkit/dsl-devkit/pull/1550) | PIPE-2 | high | small (adds `.github/scripts/test` harness) | before PIPE-PR4 | +| 5 | PIPE-PR2 | PIPE-7 | low | trivial | early, so later PIPE PRs can rely on `.commit` | +| 6 | PIPE-PR3 | PIPE-4 | medium | small | before PIPE-PR4 | +| 7 | PIPE-PR4 | PIPE-1 | high | medium | after PIPE-PR1, PIPE-PR3 | +| 8 | PIPE-PR5 → PIPE-PR6 | PIPE-6; PIPE-3 | high (PIPE-3) | large (PIPE-PR6) | PIPE-PR5 before PIPE-PR6 | +| 9 | REF-PR1 — merged as [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552) | REF-2 | low | small | first xtext.ui bump | +| 10 | REF-PR2 | REF-3, REF-9 | medium | small | none | +| 11 | REF-PR3 | REF-5 | medium | small | none | +| 12 | REF-PR4 → REF-PR5 | REF-4, REF-6; REF-1 | medium (REF-1) | medium | REF-PR5 never without REF-PR4 | +| 13 | STO-PR0 → STO-PR1 | STO-1, STO-3 | medium | harness medium, fix one line | PR0 before PR1 | +| 14 | STO-PR2 | STO-2 | low | medium | after STO-PR1; before STO-PR3 | +| 15 | STO-PR3 | STO-4, STO-5 | medium | medium | after STO-PR2 | +| 16 | RO-PR1 | RO-1, RO-3 | medium | small | none | +| 17 | PIPE-PR7 | PIPE-5 | medium | medium | after PIPE-PR3, PIPE-PR6; last PIPE PR | +| 18 | LDR-PR2 | LDR-2 | low | small | before STO-PR5 | +| 19 | STO-PR5 | STO-8 | low | small | after LDR-PR2 and STO-PR3 | +| 20 | STO-PR4 | STO-6 (+ optional STO-A2) | low | small | independent of STO-PR3/PR5 | +| 21 | LDR-PR3 | LDR-3 | low | small | none | +| 22 | TRIE-PR3 | TRIE-3, TRIE-12 | low | small | after TRIE-PR2 (or combined) | +| 23 | TRIE-PR4, TRIE-PR5 | TRIE-4; TRIE-10 | low | small | independent | +| 24 | TRIE-PR6 | TRIE-5, TRIE-6, TRIE-16 | low | small | before TRIE-PR11 | +| 25 | TRIE-PR7, TRIE-PR8 | TRIE-7; TRIE-8 | low | small | independent (may be combined) | +| 26 | TRIE-PR9 | TRIE-12, TRIE-13, TRIE-14, TRIE-15 | low | medium | after TRIE-PR2, TRIE-PR3 | +| 27 | REF-PR6 | REF-7, REF-8 | low | medium (model extension first) | after REF-PR5 | +| 28 | RO-PR2 → RO-PR3 | optional: RO-A1 (medium), RO-A2; then RO-2, RO-4, RO-5, RO-6 | low (RO-A1 medium) | medium | RO-PR2 before RO-PR3 | +| 29 | RO-PR4 | RO-8 | low | trivial | before RO-PR5 | +| 30 | REF-PR7 | optional: REF-A1, REF-A2 | low | small | after REF-PR4 | +| 31 | TRIE-PR10, TRIE-PR11 | TRIE-9; TRIE-11 | low | issue first | TRIE-PR10 after TRIE-PR3; TRIE-PR11 after TRIE-PR6 | +| 32 | RO-PR5 | RO-7 | low | small | last: needs every campaign test registered or excluded | + +**Upstream reports** (optional; each needs the user's explicit approval before posting): one Xtext issue for LDR-1 + LDR-3 (plus the interrupt-as-missing-result shape of LDR-2), and one for REF-2 + REF-6 + the REF-5 upstream variant. No other finding is shared upstream. + +## 6. Open questions + +1. **Tycho confirmation: resolved.** Every `@Disabled` method was run enabled under the Tycho aggregator: all 21 fail on master as documented; with the fix patches (plus bundle bumps) all pass except TRIE-9 (by design) and the two `SLT` methods (RO-1 has no patch yet). The 3 guard methods in `PAEDLT` and 1 in `SLT` pass on master. +2. **Design decisions before a fix:** TRIE-9 (options a/b/c in TRIE-PR10) and TRIE-11 (copy vs. documented aliasing in TRIE-PR11). +3. **LDR-2 model placement.** The chosen fix location (the catch that immediately follows the failed poll) is modelled by neither TLA+ nor Lean. Before LDR-PR2, move the TLA+ FIX-2 abort into `Catch` guarded by `bintr`, rerun `check_all.sh`, and compare with `expected.txt`. +4. **Model notes lag this catalogue.** `formal/find-refs/tla/NOTES.md` still says the REF-7 race exists upstream; this catalogue records the judge's correction (DDK-only) but the NOTES files were not edited. +5. **Observations** in [4.3](#43-verified-observations) are verified (phase C); STO-A1 still needs a decision to catalogue it before any PR claims it. diff --git a/formal/README.md b/formal/README.md new file mode 100644 index 0000000000..271942875d --- /dev/null +++ b/formal/README.md @@ -0,0 +1,85 @@ +# Formal verification of DDK: analysis and status + +> **Reference only, not for merge.** This directory records a lightweight formal-methods campaign on DDK: models, results, reproducible checks, and a catalogue of the defects found, with fix plans. No production code is changed; fixes are planned as separate PRs in [BUGS.md](BUGS.md). + +## Method + +The approach is targeted model-checking used as a bug finder, not full verification. + +1. **Pick a target.** Choose a small, well-bounded, stateful or algorithmic part of the code, such as a race-prone handoff, an index structure or the release pipeline. +2. **Model it blind.** Independent agents write a faithful model in **TLA+** (checked with TLC) and in **Lean 4** (exhaustive bounded search plus proofs). The agents are told only generic expectations, never a suspected bug. The model mirrors the code as written; it does not fix it. +3. **Check it.** Each property is checked and each counterexample is mapped back to `file:line`. +4. **Check the model can catch bugs.** A fixed model must pass, a planted bug must be caught, and witness states must be reachable. Ablations show that each fix is needed. +5. **Verify adversarially.** Every finding goes to 3 independent skeptics, each looking at code-path correctness, the threading or environment assumptions, and the real impact. A finding is kept if at least 2 of 3 uphold it; severity is set by the impact that DDK or its downstream consumers can actually reach. +6. **Confirm in Java.** Where a deterministic failing-first test is possible, one is written against the real classes and run under the Tycho aggregator. + +## Targets and status + +| # | Target | Code | Tools | Difficulty | Findings (confirmed high / med / low) | +|---|---|---|---|---|---| +| 1 | Parallel resource loader and builder cluster loop | `ParallelResourceLoader`, `MonitoredClusteringBuilderState` | TLA+, Lean | easy | LDR: 0 / 1 / 2 | +| 3 | Find-references UI batching | `FastReferenceSearchResultContentProvider` | TLA+, Lean | easy | REF: 0 / 3 / 6 | +| 5 | Binary-model storage executor vs the shared `sources` set | `MonitoredClusteringBuilderState` | TLA+, Lean | medium | STO: 0 / 2 / 4 (+1 plausible, 1 refuted) | +| 2 | Qualified-name segment trie and pattern bounds | `QualifiedNameSegmentTreeLookup`, `QualifiedNamePattern`, `PatternAwareEObjectDescriptionLookUp` | Lean (primary), TLA+ | medium-hard | TRIE: 2 / 0 / 14 | +| 4 | Release and snapshot publish pipeline | `.github/workflows/{release,snapshot}.yml`, `.github/scripts/*` | TLA+ (primary), Lean | hard | PIPE: 3 / 2 / 2 | +| – | Code-read findings (formatter comparator, orphaned tests) | `ExtendedFormattingConfigBasedStream`, test aggregator | code read + scripts | – | RO: 0 / 1 / 7 | + +**Total: 51 catalogued findings.** 49 are confirmed (5 high, 9 medium, 35 low), 1 is plausible and 1 was refuted. A further 9 adjacent observations were verified by three skeptics and are counted separately: all 9 confirmed (1 medium, 7 low, 1 not a defect). The full catalogue, with traces, verification votes and fix plans, is in [BUGS.md](BUGS.md). + +**The most important findings:** +- **TRIE-1** ([#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551)): case-sensitive pattern queries in `PatternAwareEObjectDescriptionLookUp` always return nothing, because the pattern is matched against itself. This is proved for all inputs in Lean. +- **TRIE-2:** a top-level `*`/`**` pattern misses almost every name. +- **LDR-1** ([#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553)): a load poll that waits the full 300 s timeout with no result aborts the whole build as if the user had cancelled it. Lowered to medium after the skeptics: the trigger is rare. +- **PIPE-1/2/3** (PIPE-2: [#1550](https://github.com/dsldevkit/dsl-devkit/pull/1550)): maintenance lines cannot be released, the next version is derived from the wrong tag line, and a release tag can be left permanently without a release. +- **REF-1:** a UI deadlock between the Reset handler's `syncExec` and `removeListener`. Its severity was lowered to medium because it needs a narrow timing window. + +**Fix PRs:** +- [#1550](https://github.com/dsldevkit/dsl-devkit/pull/1550) PIPE-2 (merged) +- [#1551](https://github.com/dsldevkit/dsl-devkit/pull/1551) TRIE-1 (merged) +- [#1552](https://github.com/dsldevkit/dsl-devkit/pull/1552) REF-2 (merged) +- [#1553](https://github.com/dsldevkit/dsl-devkit/pull/1553) LDR-1 (draft) + +## Models + +| Target | TLA+ (lines) | Lean (lines) | Lean proofs for all sizes | +|---|---|---|---| +| parallel-loader | 620 | 686 | `P1_fixTimeout`: the counter invariant holds for all sizes after the fix | +| find-refs | 483 | 723 | `no_lost_root`: no lost update in the fixed design | +| binary-storage | 504 | 617 | P1+P3: a URI is binary-loadable only once its binary is written (fixed model) | +| trie | 701 | 1,679 | the root causes of TRIE-1/2/3/4/5 and the correctness of the `'\u0000'` successor fix | +| pipeline | 517 | 792 | none; bounded `native_decide` results only | + +Each directory's `NOTES.md` covers the modelled assumptions, the properties, how each was checked (a proof or a bounded check), the state counts, and the traces. The notes sometimes cite run outputs (`*.out`, `logs/`, `out/`, `matrix*.log`). Those files are regenerated by `check.sh` and not committed; the committed `tla/expected.txt` files hold the verdicts. + +## How to reproduce + +```sh +# once: TLA+ tools (v1.7.4, SHA-256 936a262061c914694dfd669a543be24573c45d5aa0ff20a8b96b23d01e050e88) +gh release download v1.7.4 -R tlaplus/tlaplus -p tla2tools.jar -D formal/.tools +# Lean: elan with toolchain leanprover/lean4:v4.35.0-rc2 (no Mathlib, no dependencies) + +formal/check.sh # every model; about 12-15 min +formal/check.sh --full # adds the slow liveness and large runs (about 45 min more) +formal/check.sh --only=trie +``` + +What `check.sh` does: +- **TLA+:** runs each target's matrix and compares every run's verdict with `tla/expected.txt`. For the code as written, the expected verdict is VIOLATED for each confirmed finding. Fixed models must pass. Ablations, planted bugs and witnesses must be violated. +- **Lean:** builds each project, rejects any `sorry` or `admit`, and runs `#print axioms` on every theorem listed in `lean/theorems.txt`. Kernel proofs may use only `propext`, `Classical.choice` and `Quot.sound`. `native_decide` results may also use the compiler-trust axioms. +- **Orphaned tests:** runs `readonly/orphans/check-test-reachability.sh`, which is expected to report the test classes that never run. + +**Java tests.** Each failing-first test is registered in `XtextTestSuite` or `XtextUiTestSuite` and annotated `@Disabled("Documents …")`, so CI stays green. To see a bug, remove the annotation and run the aggregator. A fix PR removes the annotation together with the fix. Reference fixes are in `f1-fix.patch` (LDR-1), `find-refs/fix-cme.patch` (REF-2) and `trie/lean/fix-plan.patch` (TRIE). + +## Local validation done for this PR + +- `formal/check.sh --clean` passes: 180 TLC runs across 5 models, 5 Lean builds, 70 kernel proofs and 54 `native_decide` results, and the orphan check. +- **Tests enabled on master:** all 21 `@Disabled` methods fail exactly as documented, and the 4 guard methods pass. +- **With the reference patches and bundle bumps applied:** every test passes except TRIE-9 (by design) and RO-1 (no patch yet). Five `QualifiedNamePatternTest` methods that assert the old `"!"` bound fail and must be updated by the TRIE fix PRs. +- **Tests disabled:** the full `mvn clean verify` matches master exactly, including 4 pre-existing environment-specific failures on the local JDK 27 build that CI (Java 21) does not show. `checkstyle:check`, `pmd:check` and the line-ending check pass. + +## Limits + +- **Models are abstractions.** Every counterexample was checked against the real code, and the Java tests confirm the deterministic ones. Races marked CONFIRMED rely on the model plus code or bytecode reading. +- **Bounds are small.** TLC and the Lean searches cover small configurations exhaustively, typically 1–3 threads and 2–5 names or URIs. The all-sizes claims are only the Lean proofs listed above. +- **STO is downstream-only.** DDK's own target platforms don't enable binary-model storage, so those findings affect downstream products only; their severities are capped accordingly. +- **Campaign log.** [REPORT.md](REPORT.md) is the chronological log of rounds 1–2, kept as it was written. [BUGS.md](BUGS.md) is authoritative wherever the two differ, for example on severities. diff --git a/formal/REPORT.md b/formal/REPORT.md new file mode 100644 index 0000000000..4f8fa9ac4b --- /dev/null +++ b/formal/REPORT.md @@ -0,0 +1,102 @@ +# Formal-methods smoke test: ParallelResourceLoader (2026-09-25) + +## Question +Does Cherny-style modelling find real bugs in DDK, done blind and with each tool? The modelling agents were never told about the known bug (F1). + +## Results +| | TLA+ (TLC 1.7.4) | Lean 4 (v4.35.0-rc2, no Mathlib) | +|---|---|---| +| Found F1 blind | yes | yes | +| New findings | F2, F3 | F2 | +| False alarms | 0 | 0 | +| Model size | 306 lines (+314 for the fixed variant) | 686 lines (Model 212, Check 116, Results 104, Proof 250) | +| Checking | Exhaustive TLC; largest run 37k states in 3 s; the full matrix takes ~45 s | Exhaustive BFS to a fixpoint over 80 configurations (38.5k states); `native_decide` theorems; one inductive **proof** of P1 for all sizes once F1 is fixed | +| Sanity: fixed model passes | yes | yes | +| Sanity: planted bug caught | yes | yes | +| Agent wall time | ~11 min | ~15 min | + +PRL = `ParallelResourceLoader.java`; MCBS = `MonitoredClusteringBuilderState.java`. + +## Findings +- **F1 (confirmed by a Java test):** a single `poll()` timeout decrements `toProcess` (PRL:220-221). `hasNext()` then returns false while a URI is still queued, and MCBS:531-536 aborts the build with `OperationCanceledException` although nobody cancelled. The default timeout is 300 s, so this happens whenever one resource takes longer than that to load. Upstream Xtext's `ParallelResourceLoader` has the same code. + - Fix: decrement only when `result != null`. +- **F2 (confirmed by reading the code; needs an external interrupt):** the builder thread is interrupted and the queue is `LinkedBlockingQueue` or `ArrayBlockingQueue`. + - `poll` throws `InterruptedException` even though a result is waiting. The interrupt flag is restored and a fake timeout is thrown (PRL:222-230). + - `pollForCancellation` uses `sleepUninterruptibly` (MCBS:1359-1365), so the flag survives. + - The loop spins every ~5 s until the user cancels. `SynchronousQueue` (queueSize 0) is not affected. + - Fix: treat an interrupt as a cancel. +- **F3 (TLA+ only; depends on whether a load swallows the interrupt):** after `cancel()`, a worker whose load cleared the interrupt flag blocks forever in `put()` (PRL:367) with `SynchronousQueue` or a full `ArrayBlockingQueue`. The result is a leaked thread. + - Fix: a volatile `cancelled` flag checked by a timed `offer` loop. + +## Java regression test (F1) +`com.avaloq.tools.ddk.xtext.test/.../builder/resourceloader/ParallelResourceLoaderTest`, registered in `XtextTestSuite`. +- **Master:** FAILS as predicted: `hasNext()` is false after the timeout (`ParallelResourceLoaderTest.java:76`). +- **With the one-line fix** (plus the required builder bundle bump 17.3.1 to 17.3.2; saved as `formal/f1-fix.patch`, not applied): PASSES. The other failures are unchanged, and `CheckQuickfixTest.testBulkApplyingQuickfix` failed once, a known flake. +- **Unrelated failures on master in the same local run (JDK 27):** `Export/ScopeExpressionCodeGenerationTest.testNestedArithmetic`, `IssueExpressionGenerationTest.testDynamicMarkerFeature` and `CheckApiAccessValidationsTest.testNonAvaloqTypeAccessable`. + +## Takeaways +- Both tools found the known bug blind within about 15 minutes and raised no false alarms. Both also surfaced a new, real livelock (F2). +- TLA+ gives more per line for concurrency: its model is half the size, supports several clusters and liveness out of the box, and found F3 as well. +- Lean's strength is the all-sizes proof that the fix is right (P1), which TLC can't give. +- A practical recipe: TLA+ to find bugs, Lean to prove the fix when the invariant matters. + +--- + +# Round 2: find-references batching (#3) and binary storage executor (#5) + +Four blind agents ran in parallel, one per target per tool. None was told about any suspected bug. For each finding, "TLA+" and "Lean" give that finding's number in the tool's own `NOTES.md`; "–" means that tool didn't report it. + +## #3 FastReferenceSearchResultContentProvider (FRSRCP) +| Finding | TLA+ | Lean | Verified by me | +|---|---|---|---| +| Lost update: the updater clears `isUIUpdateScheduled` (L220) after its unlocked `isEmpty()` check (L217). A node added in that window is never shown. Upstream Xtext clears the flag *first*. | F1 | F1 | yes (javap of upstream `UIUpdater`) | +| **UI deadlock**: the Reset handler calls `syncExec` (L178) while `fireEvent` holds `listeners`. The UI thread switching to another search calls `removeListener` (L113) on the same lock. | F2 | F2 | yes (javap: `fireEvent` and `removeListener` both lock `listeners`) | +| A search's node appears in another search's view: `inputChanged` clears `rootNodes` (L111) before `removeListener` (L113). | F3 | F3 | code | +| Duplicate root / lost references: unguarded get-then-put in `resourceNode` (L158-161). | F4 | F5 | code | +| Duplicate reference row | F5 | (in F5) | code | +| **CME in `inputChanged`**: iterates the live `getMatchingReferences()` list (L118) while the search thread appends to it. | F6 | F4 | **Java test** | + +Held: nothing from before a Reset is shown, and the viewer shows one root per URI. + +## #5 Binary storage executor (MonitoredClusteringBuilderState = MCBS) +| Finding | TLA+ | Lean | Verified by me | +|---|---|---|---| +| Premature removal: MCBS:656 removes the URI from `sources` right after submitting the store, before the binary exists. The worker's own removal at L754 is the correct one. The load-failure path also removes the URI with no store at all. | B1 | B2 | code | +| Next cluster's loaders start (L668-669) before `clearResourceSet` awaits storage (L673→1204). They can read a half-written or stale binary, which requires the load to pull in dependencies. | B2 | B3 | code | +| After a timeout or interrupt → `shutdownNow`: running stores are neither awaited nor reported, and dropped stores have already left `sources`. Stale binaries get used and persist. Stores run on a detached resource set. | B3 | B4+B5 | code | +| **Data race**: `sources` is a plain `HashSet` (Xtext `SourceLevelURICache`) mutated by the builder thread, up to 4 storage workers and loader threads. | B5 | B1 | yes (javap) | +| Link exception → the resource is detached (L608) but still stored (L654), so its valid binary gets deleted. | B4 | – | yes (code) | +| A swallowed IOException keeps the old binary, yet the URI leaves `sources` (rare trigger). | – | B6 | code path exists | + +Held: every submitted store is either completed or dropped by `shutdownNow` (dropped ones only as a count, no URIs), and the build terminates. The `RejectedExecutionException` catch at L727 is effectively dead code. + +## Java tests (round 2) +- **`FastReferenceSearchResultContentProviderTest`** (xtext.ui.test, in `XtextUiTestSuite`): + - Master: FAILS with `ConcurrentModificationException`, as predicted. + - With the snapshot fix (`formal/find-refs/fix-cme.patch` plus the xtext.ui bundle bump 17.3.3 to 17.3.4): PASSES. + - Other failures are unchanged: the 4 known local JDK-27 failures. +- **Lint:** checkstyle + PMD report BUILD SUCCESS. +- **No other Java tests:** + - The #3 lost update and deadlock need either a seam in production code or a timing-based test. The deadlock is confirmed from bytecode. + - #5 needs a builder-state harness. Its findings are confirmed by the models and by reading the code, and B5 also from bytecode. + +## Round 2 numbers +| | refs TLA+ | refs Lean | storage TLA+ | storage Lean | +|---|---|---|---|---| +| Model size | 483 lines | 723 | 488 | 617 | +| Largest check | 50.6M states (fixed) | 1.89M (fixed), fixpoint | 3.86M (liveness at size M, with load/link failures off) | ~71k, fixpoint | +| All-sizes proof | – | `no_lost_root` (fixed) | – | P1+P3 (fixed) | +| Fixed model passes / planted bug caught / fixes each necessary | ✅ / ✅ / ✅ | ✅ / ✅ / ✅ | ✅ / ✅ / ✅ | ✅ / ✅ / – | +| False alarms | 0 | 1 (caught and fixed in the model's bookkeeping) | 0 | 0 | +| Agent time | ~25 min (+400 s large run) | ~27 min | ~50 min (plus a ~4 h stall waiting for a lost notification) | ~14 min | + +## Takeaways +- On two targets with no known bugs, the approach found **11 distinct real issues**. The two tools agreed on the core ones, and each tool contributed one unique finding on #5. +- **Most severe:** + - the find-references UI deadlock (the workbench freezes); + - the #5 premature `sources` removal combined with the `HashSet` data race, which can make builds silently link against stale binaries. +- **Tool split holds:** TLA+ scales further (tens of millions of states); Lean proves the fixed invariant for all sizes. +- **Suggested fix PRs later, in order:** + 1. find-references (deadlock + lost update + CME); + 2. binary storage (delete MCBS:656, make `sources` thread-safe, skip the store after the outer catch); + 3. the round-1 loader F1/F2. diff --git a/formal/binary-storage/lean/BinaryStorage.lean b/formal/binary-storage/lean/BinaryStorage.lean new file mode 100644 index 0000000000..0ef4139292 --- /dev/null +++ b/formal/binary-storage/lean/BinaryStorage.lean @@ -0,0 +1,4 @@ +import BinaryStorage.Model +import BinaryStorage.Check +import BinaryStorage.Results +import BinaryStorage.Proof diff --git a/formal/binary-storage/lean/BinaryStorage/Check.lean b/formal/binary-storage/lean/BinaryStorage/Check.lean new file mode 100644 index 0000000000..cfbb5d38ef --- /dev/null +++ b/formal/binary-storage/lean/BinaryStorage/Check.lean @@ -0,0 +1,103 @@ +import Std.Data.HashMap +import BinaryStorage.Model + +namespace BinaryStorage +open Std + +/-- Explored graph in BFS order: the parent chain of a state is a shortest path from `init`. -/ +structure Graph where + states : Array St := #[] + parent : Array (Option (Nat × Act)) := #[] + edges : Array (List Nat) := #[] + index : HashMap St Nat := {} + complete : Bool := true + +/-- Exhaustive BFS to a fixpoint (fuel only keeps the definition total). -/ +def explore (c : Cfg) (fuel : Nat := 5000000) : Graph := Id.run do + let s0 := init c + let mut g : Graph := { states := #[s0], parent := #[none], edges := #[[]], + index := ({} : HashMap St Nat).insert s0 0 } + let mut head := 0 + for _ in [0:fuel] do + if h : head < g.states.size then + let s := g.states[head] + let mut out : List Nat := [] + for (a, t) in succ c s do + match g.index.get? t with + | some j => out := j :: out + | none => + let j := g.states.size + g := { g with states := g.states.push t, parent := g.parent.push (some (head, a)), + edges := g.edges.push [], index := g.index.insert t j } + out := j :: out + g := { g with edges := g.edges.set! head out } + head := head + 1 + else break + if head < g.states.size then g := { g with complete := false } + return g + +def Graph.trace (g : Graph) (i : Nat) : List Act := Id.run do + let mut acc : List Act := [] + let mut k := i + for _ in [0:g.states.size] do + match g.parent[k]! with + | some (p, a) => acc := a :: acc; k := p + | none => break + return acc + +def Graph.firstViolation (g : Graph) (p : St → Bool) : Option Nat := + (List.range g.states.size).find? fun i => !p g.states[i]! + +/-- Kahn's algorithm: true iff the reachable graph is acyclic (every run is finite). -/ +def Graph.acyclic (g : Graph) : Bool := Id.run do + let n := g.states.size + let mut indeg : Array Nat := Array.replicate n 0 + for es in g.edges do + for j in es do indeg := indeg.modify j (· + 1) + let mut stack : List Nat := (List.range n).filter (indeg[·]! == 0) + let mut seen := 0 + for _ in [0:n] do + match stack with + | [] => break + | i :: rest => + stack := rest; seen := seen + 1 + for j in g.edges[i]! do + indeg := indeg.modify j (· - 1) + if indeg[j]! == 0 then stack := j :: stack + return seen == n + +/-- Non-terminal states without successors. -/ +def Graph.deadlocks (c : Cfg) (g : Graph) : Nat := + ((List.range g.states.size).filter fun i => g.edges[i]!.isEmpty && !terminal c g.states[i]!).length + +structure Report where + states : Nat + complete : Bool + acyclic : Bool + deadlocks : Nat + p1 : Option Nat + p2 : Option Nat + p3partial : Option Nat + p3stale : Option Nat + p4 : Option Nat + pEnd : Option Nat + deriving Repr + +def report (c : Cfg) : Report := + let g := explore c + let fv (p : St → Bool) : Option Nat := (g.firstViolation p).map fun i => (g.trace i).length + { states := g.states.size, complete := g.complete, acyclic := g.acyclic, deadlocks := g.deadlocks c, + p1 := fv p1, p2 := fv (p2 c), p3partial := fv p3partial, p3stale := fv p3stale, p4 := fv p4, pEnd := fv (pEnd c) } + +/-- Shortest counterexample (list of actions) for property `p`, if any. -/ +def cex (c : Cfg) (p : St → Bool) : Option (List Act) := + let g := explore c + (g.firstViolation p).map g.trace + +/-- Boolean verdict usable by `native_decide`: all five properties hold. -/ +def allHold (c : Cfg) : Bool := + let r := report c + r.complete && r.acyclic && r.deadlocks == 0 && r.p1.isNone && r.p2.isNone && + r.p3partial.isNone && r.p3stale.isNone && r.p4.isNone && r.pEnd.isNone + +end BinaryStorage diff --git a/formal/binary-storage/lean/BinaryStorage/Model.lean b/formal/binary-storage/lean/BinaryStorage/Model.lean new file mode 100644 index 0000000000..a63bf4da53 --- /dev/null +++ b/formal/binary-storage/lean/BinaryStorage/Model.lean @@ -0,0 +1,338 @@ +/-! +Model of the binary-model storage path of `MonitoredClusteringBuilderState` (MCBS) and its +interaction with the cluster loop, the DDK `ParallelResourceLoader` (PRL) threads and the shared +`SourceLevelURICache.sources` set (a plain `java.util.HashSet`, Xtext SourceLevelURICache:14). + +Instance: three URIs, all rebuilt in this build. + a (0), c (1) : cluster 1 (initial queue, installed as sources by installSourceLevelURIs) + b (2) : cluster 2 (affected by a; added to sources by queueAffectedResources MCBS:1296) + b depends on a: main-thread linking of b resolves a; optionally loading b also touches a. + +Threads: main builder thread (tid 0), cluster-1 loader (tid 1), cluster-2 loader (tid 2), +storage workers (tid 10+i), across executor generations (MCBS:871 recreates the pool). +-/ +namespace BinaryStorage + +/-- State of the binary file `.bin` on disk. `stale` = complete but from the previous build. -/ +inductive Bin | none | stale | part | fresh + deriving DecidableEq, Hashable, Repr, Inhabited + +inductive Task | idle | queued | running | done | failed | dropped | discarded + deriving DecidableEq, Hashable, Repr, Inhabited + +/-- Phases of one `doStoreBinaryResource` (MCBS:740-765). -/ +inductive Ph | ser | wr | wrote | rmB + deriving DecidableEq, Hashable, Repr, Inhabited + +structure Job where + u : Nat + ph : Ph + intr : Bool -- interrupted by shutdownNow + deriving DecidableEq, Hashable, Repr, Inhabited + +structure W where + gen : Nat + started : Bool + job : Option Job + deriving DecidableEq, Hashable, Repr, Inhabited + +/-- An in-flight (non-atomic) operation on the shared HashSet. -/ +structure Op where + tid : Nat + wr : Bool + racy : Bool + deriving DecidableEq, Hashable, Repr, Inhabited + +/-- Loader micro-steps: contains() begin/end on the sources set, then publish the result. -/ +inductive LA | chkB (x : Nat) | chkE (x : Nat) | pub (x : Nat) + deriving DecidableEq, Hashable, Repr, Inhabited + +/-- Main-thread program instructions. -/ +inductive I + | next (u : Nat) | store (u : Nat) | rmB (u : Nat) | rmE (u : Nat) + | addB (u : Nat) | addE (u : Nat) | startL2 | shut | awaitT | waitActive | recreate | clear + | resB (u : Nat) | resE (u : Nat) | done + deriving DecidableEq, Hashable, Repr, Inhabited + +def uA : Nat := 0 +def uC : Nat := 1 +def uB : Nat := 2 + +structure Cfg where + nW : Nat := 1 -- BINARY_STORAGE_EXECUTOR_PARALLELISM (scaled down) + qcap : Nat := 1 -- BINARY_STORAGE_EXECUTOR_QUEUE_CAPACITY (scaled down) + timeout : Bool := true -- awaitTermination may time out (1 min) or be interrupted + ioFail : Bool := false -- writeResource may throw IOException (swallowed by Xtext) + touchDep : Bool := true -- loading b in a loader thread may load a into the local set + fixed : Bool := false -- apply the candidate fix + plant : Bool := false -- planted bug (on top of `fixed`): remove from sources before writing + initBin : Bin := .stale -- previous build left a complete binary + raceNondet : Bool := true -- racing HashSet ops may miss a key / lose a write (false: only flag the race) + deriving Repr + +structure St where + pc : Nat + inl : Option Job -- CallerRunsPolicy: main runs the store itself + ops : List Op + inSrc : List Bool + bin : List Bin + task : List Task + rs : List Bool -- resource currently in the main resource set + loaded : List Bool -- loader published the resource + l1 : List LA + l2 : List LA + l2on : Bool + gen : Nat + shut : Bool + queue : List Nat + workers : List W + reported : Nat -- "{} tasks not processed" (MCBS:880) + setRace : Bool + partialLd : Bool + partialMain : Bool + staleLd : Bool + staleMain : Bool + deriving DecidableEq, Hashable, Repr, Inhabited + +abbrev Act := String + +def storeSeq (c : Cfg) (u : Nat) : List I := + if c.fixed then [.next u, .store u] else [.next u, .store u, .rmB u, .rmE u] + +def awaitSeq (c : Cfg) : List I := + if c.fixed then [.shut, .awaitT, .waitActive, .recreate] else [.shut, .awaitT, .recreate] + +/-- Main-thread program, in the order of MCBS.doUpdate L529-690 (bug) or the fixed order. -/ +def prog (c : Cfg) : List I := + let cluster1 := storeSeq c uA ++ storeSeq c uC + let qa : List I := if c.fixed then [.addB uB] else [.addB uB, .addE uB] + let boundary := + if c.fixed then qa ++ awaitSeq c ++ [.clear, .startL2] + else qa ++ [.startL2] ++ awaitSeq c ++ [.clear] + let res : List I := if c.fixed then [.resB uA] else [.resB uA, .resE uA] + let cluster2 := [.next uB] ++ res ++ (storeSeq c uB).drop 1 + cluster1 ++ boundary ++ cluster2 ++ awaitSeq c ++ [.done] + +def loadSeq (c : Cfg) (x : Nat) : List LA := + if c.fixed then [.chkB x] else [.chkB x, .chkE x] + +def init (c : Cfg) : St := + { pc := 0, inl := none, ops := [], + inSrc := [true, true, false], bin := [c.initBin, c.initBin, c.initBin], + task := [.idle, .idle, .idle], rs := [false, false, false], loaded := [false, false, false], + l1 := loadSeq c uA ++ [.pub uA] ++ loadSeq c uC ++ [.pub uC], + l2 := loadSeq c uB ++ (if c.touchDep then loadSeq c uA else []) ++ [.pub uB], + l2on := false, gen := 0, shut := false, queue := [], + workers := (List.range c.nW).map fun _ => { gen := 0, started := false, job := none }, + reported := 0, setRace := false, partialLd := false, partialMain := false, + staleLd := false, staleMain := false } + +/-! ### Shared-set operations (non-atomic unless the fix makes the set concurrent) -/ + +def opBegin (s : St) (tid : Nat) (wr : Bool) : St := + let clash (o : Op) := o.tid != tid && (o.wr || wr) + let conflict := s.ops.any clash + { s with ops := s.ops.map (fun o => if clash o then { o with racy := true } else o) ++ [⟨tid, wr, conflict⟩], + setRace := s.setRace || conflict } + +def opEnd (c : Cfg) (s : St) (tid : Nat) : Bool × St := + match s.ops.find? (·.tid == tid) with + | some o => (o.racy && c.raceNondet, { s with ops := s.ops.filter (·.tid != tid) }) + | none => (false, s) + +/-- Possible results of `contains(x)`; a read racing a structural modification (HashMap resize + installs the new, empty table before transferring) may miss a present key. -/ +def containsRes (s : St) (x : Nat) (racy : Bool) : List Bool := + if racy then (if s.inSrc[x]! then [true, false] else [false]) else [s.inSrc[x]!] + +/-- Possible results of a write to the set: a racing write may be lost. -/ +def writeRes (s : St) (x : Nat) (v : Bool) (racy : Bool) : List St := + let applied := { s with inSrc := s.inSrc.set x v } + if racy && s.inSrc[x]! != v then [applied, s] else [applied] + +/-- A load of `x` after `shouldLoadFromStorage` answered "is source" = `res` + (ResourceStorageFacade:49-54 -> StorageAwareResource:82-88). -/ +def readOutcome (s : St) (x : Nat) (isMain : Bool) (res : Bool) : St := + if res then s else + match s.bin[x]! with + | .part => if isMain then { s with partialMain := true } else { s with partialLd := true } + | .stale => if isMain then { s with staleMain := true } else { s with staleLd := true } + | _ => s + +def setTask (s : St) (u : Nat) (t : Task) : St := { s with task := s.task.set u t } +def setBin (s : St) (u : Nat) (b : Bin) : St := { s with bin := s.bin.set u b } + +/-! ### One storage job (`doStoreBinaryResource`) executed by thread `tid` -/ + +def jobSteps (c : Cfg) (s : St) (tid : Nat) (j : Job) : List (Act × St × Option Job) := + let u := j.u + match j.ph with + | .ser => + if !s.rs[u]! then + [("store: resource detached, save throws -> deleteStorage (DLRSF:77-79, MCBS:765)", + setTask (setBin s u .none) u .failed, none)] + else if c.plant then + [("PLANTED: sources.remove before write", setBin { s with inSrc := s.inSrc.set u false } u .part, + some { j with ph := .wr })] + else + [("store: generateFile starts writing binary (ResourceStorageFacade:104)", setBin s u .part, + some { j with ph := .wr })] ++ + (if c.ioFail then + (if c.fixed then [("store: IOException, fixed: delete storage, keep in sources", setTask (setBin s u .none) u .failed, none)] + else [("store: writeResource IOException swallowed, old binary kept (ResourceStorageFacade:97-103)", s, + some { j with ph := .wrote })]) + else []) + | .wr => + [("store: generateFile completes", setBin s u .fresh, some { j with ph := .wrote })] ++ + (if j.intr then [("store: interrupted write fails -> deleteStorage (DLRSF:77-79)", + setTask (setBin s u .none) u .failed, none)] else []) + | .wrote => + if c.plant then [("store done", setTask s u .done, none)] + else if c.fixed then + [("store: sources.remove after complete write (concurrent set)", + setTask { s with inSrc := s.inSrc.set u false } u .done, none)] + else [("store: sources.remove begins (MCBS:754)", opBegin s tid true, some { j with ph := .rmB })] + | .rmB => + let (racy, s') := opEnd c s tid + (writeRes s' u false racy).map fun t => ("store: sources.remove ends (MCBS:754)", setTask t u .done, none) + +/-! ### Successor function -/ + +def mainSteps (c : Cfg) (s : St) : List (Act × St) := + match s.inl with + | some j => + (jobSteps c s 0 j).map fun (a, t, j') => ("main(CallerRuns) " ++ a, { t with inl := j' }) + | none => + let p := prog c + let adv (t : St) : St := { t with pc := t.pc + 1 } + match p[s.pc]? with + | none => [] + | some ins => + match ins with + | .done => [] + | .next u => + if s.loaded[u]! then [("main: loadOperation.next(), addResource (MCBS:537)", adv { s with rs := s.rs.set u true })] + else [] + | .store u => + let s := adv s + if s.shut then [("main: execute on shut-down pool, CallerRuns discards silently (MCBS:726)", setTask s u .discarded)] + else match s.workers.findIdx? (fun w => w.gen == s.gen && !w.started) with + | some i => + [("main: execute -> new core worker (MCBS:726)", + setTask { s with workers := s.workers.set i { gen := s.gen, started := true, job := some ⟨u, .ser, false⟩ } } u .running)] + | none => + if s.queue.length < c.qcap then + [("main: execute -> queued (MCBS:726)", setTask { s with queue := s.queue ++ [u] } u .queued)] + else + [("main: queue full, CallerRunsPolicy runs store inline (MCBS:726)", + setTask { s with inl := some ⟨u, .ser, false⟩ } u .running)] + | .rmB _ => [("main: sources.remove(changedURI) begins (MCBS:656)", adv (opBegin s 0 true))] + | .rmE u => + let (racy, s') := opEnd c s 0 + (writeRes s' u false racy).map fun t => ("main: sources.remove(changedURI) ends (MCBS:656)", adv t) + | .addB u => + if c.fixed then [("main: queueAffectedResources sources.add (concurrent set)", adv { s with inSrc := s.inSrc.set u true })] + else [("main: queueAffectedResources sources.add begins (MCBS:1296)", adv (opBegin s 0 true))] + | .addE u => + let (racy, s') := opEnd c s 0 + (writeRes s' u true racy).map fun t => ("main: queueAffectedResources sources.add ends (MCBS:1296)", adv t) + | .startL2 => [("main: next cluster loadOperation.load(queue) starts loaders (MCBS:668-669)", adv { s with l2on := true })] + | .shut => [("main: binaryStorageExecutor.shutdown() (MCBS:826)", adv { s with shut := true })] + | .awaitT => + let cur := s.workers.filter (·.gen == s.gen) + let terminated := s.queue.isEmpty && cur.all (·.job.isNone) + if terminated then [("main: awaitTermination -> true (MCBS:837)", adv s)] + else if c.timeout then + -- fix (d): the not-processed stores' outdated binaries are deleted + let s1 := s.queue.foldl (fun t u => let t := setTask t u .dropped; if c.fixed then setBin t u .none else t) s + [("main: awaitTermination times out/interrupted -> shutdownNow (MCBS:837,856,879)", + adv { s1 with queue := [], reported := s.reported + s.queue.length, + workers := s.workers.map fun (w : W) => + if w.gen == s.gen then { w with job := w.job.map fun j => { j with intr := true } } else w })] + else [] + | .waitActive => + if (s.workers.filter (·.gen == s.gen)).all (·.job.isNone) then [("main(fixed): wait for active stores", adv s)] else [] + | .recreate => + [("main: binaryStorageExecutor = makeBinaryStorageExecutor() (MCBS:871)", + adv { s with gen := s.gen + 1, shut := false, + workers := s.workers ++ (List.range c.nW).map fun _ => { gen := s.gen + 1, started := false, job := none } })] + | .clear => [("main: clearResourceSet clears resource set (MCBS:1207)", adv { s with rs := [false, false, false] })] + | .resB x => + if s.rs[x]! then [("main: dependency already in resource set", { s with pc := s.pc + (if c.fixed then 1 else 2) })] + else if c.fixed then + [("main: resolve dependency -> getResource(load=true), shouldLoadFromStorage", + adv (readOutcome { s with rs := s.rs.set x true } x true s.inSrc[x]!))] + else [("main: resolveLazyCrossReferences loads dependency; shouldLoadFromStorage contains() begins (MCBS:564, RSF:51)", + adv (opBegin s 0 false))] + | .resE x => + let (racy, s') := opEnd c s 0 + (containsRes s' x racy).map fun r => + ("main: contains() ends -> " ++ (if r then "parse source" else "load binary (StorageAwareResource:86)"), + adv (readOutcome { s' with rs := s'.rs.set x true } x true r)) + +def loaderSteps (c : Cfg) (s : St) (tid : Nat) (l : List LA) (upd : St → List LA → St) : List (Act × St) := + let name := if tid == 1 then "loader1: " else "loader2: " + match l with + | [] => [] + | a :: rest => + match a with + | .pub x => [(name ++ "publish loaded resource (PRL:323)", upd { s with loaded := s.loaded.set x true } rest)] + | .chkB x => + if c.fixed then [(name ++ "load, shouldLoadFromStorage (concurrent set)", upd (readOutcome s x false s.inSrc[x]!) rest)] + else [(name ++ s!"load uri#{x}: shouldLoadFromStorage contains() begins (PRL:174, RSF:51)", upd (opBegin s tid false) rest)] + | .chkE x => + let (racy, s') := opEnd c s tid + (containsRes s' x racy).map fun r => + (name ++ s!"contains(uri#{x}) ends -> " ++ (if r then "parse source" else "load binary"), + upd (readOutcome s' x false r) rest) + +def workerSteps (c : Cfg) (s : St) : List (Act × St) := Id.run do + let mut out : List (Act × St) := [] + for i in [0:s.workers.length] do + let w := s.workers[i]! + match w.job with + | some j => + for (a, t, j') in jobSteps c s (10 + i) j do + out := (s!"worker{i}(gen{w.gen}): " ++ a, { t with workers := t.workers.set i { w with job := j' } }) :: out + | none => + if w.gen == s.gen && w.started then + match s.queue with + | u :: q => + out := (s!"worker{i}: takes store of uri#{u} from queue", + setTask { s with queue := q, workers := s.workers.set i { w with job := some ⟨u, .ser, false⟩ } } u .running) :: out + | [] => out := out + return out.reverse + +def succ (c : Cfg) (s : St) : List (Act × St) := + mainSteps c s ++ workerSteps c s ++ + loaderSteps c s 1 s.l1 (fun t l => { t with l1 := l }) ++ + (if s.l2on then loaderSteps c s 2 s.l2 (fun t l => { t with l2 := l }) else []) + +def mainDone (c : Cfg) (s : St) : Bool := s.inl.isNone && (prog c)[s.pc]? == some .done + +def terminal (c : Cfg) (s : St) : Bool := + mainDone c s && s.workers.all (·.job.isNone) && s.l1.isEmpty && (s.l2.isEmpty || !s.l2on) + +/-! ### Properties -/ + +/-- P1: a rebuilt URI is binary-loadable (not in sources and a binary exists) only when complete. -/ +def p1 (s : St) : Bool := + (List.range 3).all fun u => s.task[u]! == .idle || s.inSrc[u]! || s.bin[u]! == Bin.none || s.bin[u]! == Bin.fresh + +/-- P2: when the build returns, every submitted store completed or was reported as not processed. -/ +def p2 (c : Cfg) (s : St) : Bool := + !mainDone c s || s.task.all fun t => t == .idle || t == .done || t == .failed || t == .dropped + +/-- P3: nobody reads a binary that is being written (partial) ... -/ +def p3partial (s : St) : Bool := !s.partialLd && !s.partialMain +/-- ... nor an outdated binary of a URI rebuilt in this build. -/ +def p3stale (s : St) : Bool := !s.staleLd && !s.staleMain +/-- P5: when everything has stopped, no rebuilt URI is left with an outdated binary on disk + (later builds load every non-source URI from its binary). -/ +def pEnd (c : Cfg) (s : St) : Bool := + !terminal c s || (List.range 3).all fun u => s.task[u]! == .idle || s.bin[u]! != .stale + +/-- P4: no unsynchronised concurrent access to the non-thread-safe sources HashSet. -/ +def p4 (s : St) : Bool := !s.setRace + +end BinaryStorage diff --git a/formal/binary-storage/lean/BinaryStorage/Proof.lean b/formal/binary-storage/lean/BinaryStorage/Proof.lean new file mode 100644 index 0000000000..5052ef9cec --- /dev/null +++ b/formal/binary-storage/lean/BinaryStorage/Proof.lean @@ -0,0 +1,120 @@ +import BinaryStorage.Model + +/-! +All-sizes inductive-invariant proof of P1/P3 for the FIXED protocol. + +Abstraction: each URI carries its own store-lifecycle phase; the executor (any number of +workers, any queue capacity, CallerRunsPolicy, shutdown/shutdownNow, any number of clusters and +executor generations) only decides *which* URI steps next, so the global system is +"any URI takes one protocol step at a time", over an unbounded URI space (`Nat → U`). +The sources set is concurrent in the fix, so each set operation is one atomic step. +-/ +namespace BinaryStorage.Generic + +inductive GPh | idle | queued | ser | writing | written | done | dropped + deriving DecidableEq, Repr + +structure U where + rebuilt : Bool + inSrc : Bool + bin : Bin + ph : GPh + +/-- One step of the fixed per-URI protocol. -/ +inductive FStep : U → U → Prop + /-- installSourceLevelURIs / queueAffectedResources: URI becomes (re)built, added to sources. -/ + | mark (u : U) : FStep u { u with rebuilt := true, inSrc := true } + | submit (u : U) : u.rebuilt = true → u.ph = .idle → FStep u { u with ph := .queued } + | start (u : U) : u.ph = .queued → FStep u { u with ph := .ser } + /-- shutdownNow returns the task; fix deletes its outdated binary. -/ + | drop (u : U) : u.ph = .queued → FStep u { u with ph := .dropped, bin := .none } + | beginWrite (u : U) : u.ph = .ser → FStep u { u with ph := .writing, bin := .part } + /-- detached resource / IOException: fix deletes storage, keeps URI in sources. -/ + | failEarly (u : U) : u.ph = .ser → FStep u { u with ph := .done, bin := .none } + | endWrite (u : U) : u.ph = .writing → FStep u { u with ph := .written, bin := .fresh } + | abortWrite (u : U) : u.ph = .writing → FStep u { u with ph := .done, bin := .none } + /-- the only removal from sources: after the complete write. -/ + | remove (u : U) : u.ph = .written → FStep u { u with ph := .done, inSrc := false } + +/-- The code as written adds MCBS:656 — removal right after submission, in any phase. -/ +inductive BStep : U → U → Prop + | fixed (u v : U) : FStep u v → BStep u v + | mainRemove (u : U) : u.ph ≠ .idle → BStep u { u with inSrc := false } + +abbrev Sys := Nat → U + +def initial (σ : Sys) : Prop := ∀ k, (σ k).rebuilt = false ∧ (σ k).ph = .idle + +inductive Reach (R : U → U → Prop) : Sys → Prop + | init (σ : Sys) : initial σ → Reach R σ + | step (σ : Sys) (k : Nat) (v : U) : Reach R σ → R (σ k) v → + Reach R (fun j => if j = k then v else σ j) + +/-- Per-URI invariant. -/ +def Inv (u : U) : Prop := + (u.rebuilt = false → u.ph = .idle) ∧ + (u.rebuilt = true → (u.ph = .written → u.bin = .fresh) ∧ (u.inSrc = false → u.bin = .fresh ∧ u.ph = .done)) + +theorem inv_step {u v : U} (h : Inv u) (s : FStep u v) : Inv v := by + obtain ⟨h1, h2⟩ := h + cases s with + | mark => + refine ⟨by simp, fun _ => ⟨fun hw => ?_, by simp⟩⟩ + cases hr : u.rebuilt + · have := h1 hr; simp_all + · exact (h2 hr).1 hw + | submit hr hi => exact ⟨by simp_all, fun _ => ⟨by simp, fun hs => by have := (h2 hr).2 hs; simp_all⟩⟩ + | start hq => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun hs => ?_⟩⟩ + have := (h2 hr).2 hs; simp_all + | drop hq => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun hs => ?_⟩⟩ + have := (h2 hr).2 hs; simp_all + | beginWrite hq => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun hs => ?_⟩⟩ + have := (h2 hr).2 hs; simp_all + | failEarly hq => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun hs => ?_⟩⟩ + have := (h2 hr).2 hs; simp_all + | endWrite hq => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun hs => ?_⟩⟩ + have := (h2 hr).2 hs; simp_all + | abortWrite hq => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun hs => ?_⟩⟩ + have := (h2 hr).2 hs; simp_all + | remove hw => + refine ⟨fun hr => by have := h1 hr; simp_all, fun hr => ⟨by simp, fun _ => ?_⟩⟩ + exact ⟨(h2 hr).1 hw, rfl⟩ + +theorem inv_reach {σ : Sys} (h : Reach FStep σ) : ∀ k, Inv (σ k) := by + induction h with + | init σ hi => + intro k; obtain ⟨hr, hp⟩ := hi k + exact ⟨fun _ => hp, fun h => by simp_all⟩ + | step σ k v _ hs ih => + intro j + by_cases hj : j = k + · subst hj; simpa using inv_step (ih j) hs + · simpa [hj] using ih j + +/-- **P1 + P3 for all sizes (fixed protocol).** Whenever a rebuilt URI is binary-loadable + (not a source-level URI and a binary exists) the binary is completely written: no reader can + observe a partial or outdated binary, for any number of URIs, workers, clusters, interleavings. -/ +theorem fixed_safe {σ : Sys} (h : Reach FStep σ) (k : Nat) (hr : (σ k).rebuilt = true) + (hs : (σ k).inSrc = false) : (σ k).bin = .fresh := + ((inv_reach h k).2 hr).2 hs |>.1 + +/-- The same statement fails for the code as written (MCBS:656), already with one URI. -/ +theorem buggy_unsafe : ∃ σ, Reach BStep σ ∧ (σ 0).rebuilt = true ∧ (σ 0).inSrc = false ∧ (σ 0).bin = .stale := by + let σ0 : Sys := fun _ => { rebuilt := false, inSrc := false, bin := .stale, ph := .idle } + have r0 : Reach BStep σ0 := .init σ0 (fun _ => ⟨rfl, rfl⟩) + let u1 : U := { rebuilt := true, inSrc := true, bin := .stale, ph := .idle } + have r1 := Reach.step σ0 0 u1 r0 (.fixed _ _ (.mark _)) + let σ1 : Sys := fun j => if j = 0 then u1 else σ0 j + let u2 : U := { u1 with ph := .queued } + have r2 := Reach.step σ1 0 u2 r1 (.fixed _ _ (.submit _ (by simp [σ1, u1]) (by simp [σ1, u1]))) + let σ2 : Sys := fun j => if j = 0 then u2 else σ1 j + have r3 := Reach.step σ2 0 { u2 with inSrc := false } r2 (.mainRemove _ (by simp [σ2, u2])) + exact ⟨_, r3, by simp [u2, u1], by simp, by simp [u2, u1]⟩ + +end BinaryStorage.Generic diff --git a/formal/binary-storage/lean/BinaryStorage/Results.lean b/formal/binary-storage/lean/BinaryStorage/Results.lean new file mode 100644 index 0000000000..08be7f4e12 --- /dev/null +++ b/formal/binary-storage/lean/BinaryStorage/Results.lean @@ -0,0 +1,52 @@ +import BinaryStorage.Check + +/-! +Machine-checked verdicts of the exhaustive BFS (bounded instance: 3 URIs, 2 clusters, +1-2 storage workers, queue capacity 0-1, 3 executor generations). `native_decide` trusts the +compiler; the BFS reaches a fixpoint (`complete`) so each verdict covers every interleaving. +-/ +namespace BinaryStorage + +/-! ### Fixed model: every property holds, every run terminates, no deadlock -/ +theorem fixed_default : allHold { fixed := true } = true := by native_decide +theorem fixed_two_workers_iofail : allHold { fixed := true, nW := 2, ioFail := true } = true := by native_decide +theorem fixed_caller_runs : allHold { fixed := true, qcap := 0, nW := 2, ioFail := true } = true := by native_decide +theorem fixed_no_old_binary : allHold { fixed := true, initBin := .none } = true := by native_decide + +/-! ### Sanity: planted bug (remove from sources before the write) is caught -/ +theorem planted_caught : ((report { fixed := true, plant := true }).p1).isSome = true := by native_decide + +/-! ### Code as written: violations -/ +/-- P4: unsynchronised concurrent access to the sources HashSet (7 steps). -/ +theorem bug_p4 : (report {}).p4 = some 7 := by native_decide +/-- P1: URI binary-loadable before its binary is written (MCBS:656), 7 steps. -/ +theorem bug_p1 : (report {}).p1 = some 7 := by native_decide +/-- P3 (stale), with atomic set semantics and without any timeout: next-cluster loader reads + the outdated binary of a URI rebuilt in the previous cluster (21 steps). -/ +theorem bug_stale_no_timeout : + (report { raceNondet := false, timeout := false }).p3stale = some 21 := by native_decide +/-- P3 (partial), atomic set semantics: loader reads a binary being written (22 steps). -/ +theorem bug_partial : (report { raceNondet := false }).p3partial = some 22 := by native_decide +/-- P2: store still running (neither completed nor reported) when the build returns (35 steps). -/ +theorem bug_p2 : (report {}).p2 = some 35 := by native_decide +/-- P5: an outdated binary survives the build: dropped store (timeout) ... -/ +theorem bug_pEnd_timeout : ((report { raceNondet := false }).pEnd).isSome = true := by native_decide +/-- ... or a swallowed IOException, even without timeouts and races. -/ +theorem bug_pEnd_iofail : + ((report { raceNondet := false, timeout := false, ioFail := true }).pEnd).isSome = true := by native_decide +/-- Without timeouts, IOException, dependency loading in loaders and HashSet races, only the + transient P1 window and the set race itself remain. -/ +theorem bug_minimal : + let r := report { raceNondet := false, timeout := false, touchDep := false } + r.p2.isNone && r.p3partial.isNone && r.p3stale.isNone && r.pEnd.isNone && r.p4.isSome = true := by + native_decide +/-- The bounded build always terminates and never deadlocks (code as written). -/ +theorem bug_terminates : let r := report {}; r.complete && r.acyclic && r.deadlocks == 0 = true := by + native_decide +/-- CallerRunsPolicy path is exercised, and a silent discard (execute on a shut-down pool) is unreachable. -/ +theorem caller_runs_reachable_discard_unreachable : + let g := explore { qcap := 0 } + (g.states.toList.any (·.inl.isSome)) && !(g.states.toList.any (·.task.contains .discarded)) = true := by + native_decide + +end BinaryStorage diff --git a/formal/binary-storage/lean/NOTES.md b/formal/binary-storage/lean/NOTES.md new file mode 100644 index 0000000000..a911dee87c --- /dev/null +++ b/formal/binary-storage/lean/NOTES.md @@ -0,0 +1,252 @@ +# Binary-model storage: Lean model notes + +Subject: `MonitoredClusteringBuilderState` (MCBS), `com.avaloq.tools.ddk.xtext.builder`, its binary +storage executor, and the shared `SourceLevelURICache.sources` set, together with the DDK +`ParallelResourceLoader` (PRL) and Xtext 2.44 `ResourceStorageFacade` (RSF), `StorageAwareResource` +(SAR), `SourceLevelURIsAdapter`, and DDK `DirectLinkingResourceStorageFacade` (DLRSF). + +Build: `lake build` (toolchain `leanprover/lean4:v4.35.0-rc2`, no dependencies, no downloads). +A clean build takes about 8 s. Lean size: 617 lines (Model 338, Check 103, Proof 120, Results 52). +Wall time for the whole exercise: about 14 minutes. + +## Facts established from the code + +| Fact | Where | +|---|---| +| `sources` is a plain `HashSet` (not thread-safe) | Xtext `SourceLevelURICache:14` (`Sets.newHashSet()`) | +| The main resource set's adapter wraps that same set without a copy | MCBS:1536 → `SourceLevelURIsAdapter.setSourceLevelUrisWithoutCopy` | +| Every loader thread's local resource set gets the same view | PRL:168, PRL:174 | +| Every load asks `sources.contains(uri)`. If the URI is absent and a binary exists, it loads the binary | SAR:82-88 → RSF:49-54 (`doesStorageExist`); DLRSF:58-63 only adds the SKIP mode | +| Writers of `sources`: main thread `remove` right after submitting the store | MCBS:656 | +| Writers of `sources`: each storage worker `remove`s after `saveResource` (up to 4 at once) | MCBS:754 | +| Writers of `sources`: main thread `add` in `queueAffectedResources` | MCBS:1296 | +| The executor is fixed (4 threads, 15 000 queue slots) with `CallerRunsPolicy` | MCBS:181-192 | +| `awaitBinaryStorageExecutorTermination()` waits 1 minute with 0 retries. On timeout or interrupt it calls `shutdownNow()`, logs only a count, does not wait for running tasks, and replaces the executor | MCBS:803-872, MCBS:877-881 | +| The next cluster's loaders start before storage is awaited: `load(queue)` at L668-669 comes before `clearResourceSet` → await at L673/L1204 | MCBS:668-673, MCBS:1204 | +| `saveResource` swallows `IOException` from `writeResource` and leaves the old file in place. DLRSF deletes only on a thrown exception or when the resource has errors | RSF:95-104, DLRSF:70-80 | +| Nothing deletes the old binary of a URI that is being rebuilt. Only `toBeDeleted` URIs are cleaned | MCBS:452-453, MCBS:777-787 | + +## Model (`BinaryStorage/Model.lean`) + +The model covers one build with three URIs. Cluster 1 is `a` and `c`, both initial sources. +Cluster 2 is `b`: it is affected by `a`, becomes a source at L1296, and linking it resolves `a`. +Threads: +- the main builder thread, running the doUpdate program L529-690; +- a cluster-1 loader and a cluster-2 loader (PRL); +- the storage workers. Each executor generation has `nW` workers, and the model has 3 generations + because MCBS:871 recreates the pool. + +What the state tracks: +- per URI: `inSrc`, the binary state (`none | stale | part | fresh`), the store task state, + whether the resource is in the resource set, and whether it has been loaded; +- the executor: queue, `shut`, generation, and workers, including zombie workers left over from an + earlier generation; +- the CallerRuns inline job; +- in-flight HashSet operations; +- the hazard flags. + +Steps that are not atomic in the code are separate steps in the model: +- `contains`, `add` and `remove` each have a begin step and an end step; +- a store has four phases: serialize, write begins (binary becomes partial), write ends (binary + becomes fresh), and L754 removal (begin and end). + +The executor follows Java semantics: +- core worker threads are created lazily, then tasks queue until the capacity is reached; +- when the queue is full, CallerRuns runs the task on the main thread; after shutdown it silently + discards it; +- `shutdown` lets queued tasks drain; +- `awaitTermination` either returns true or times out (nondeterministically; this also stands for + an `InterruptedException`); +- `shutdownNow` drops the queued tasks and interrupts running ones, which keep running; +- an interrupted write fails, and DLRSF then deletes the storage. + +Configuration switches: + +| Switch | What it enables | +|---|---| +| `timeout` | `awaitTermination` may time out | +| `ioFail` | the IOException in `writeResource` is swallowed | +| `touchDep` | loading `b` in a loader thread also loads `a` into its local set | +| `raceNondet` | racing HashSet operations may miss a key or lose a write. With `false`, races are only flagged | +| `fixed` | applies the candidate fix | +| `plant` | adds a planted bug on top of the fix | + +Candidate fix (`fixed := true`), not applied to the Java: +1. Remove from `sources` only after the binary is completely written: drop L656 and keep L754. +2. Make `sources` a concurrent set. +3. Await storage termination before starting the next cluster's loaders. On timeout, wait for the + running tasks as well. +4. Delete the outdated binary of any store that did not complete (dropped, IOException, or + detached resource). + +Assumptions and abstractions: +- A loader is modelled as one sequential thread, where the code uses N threads. +- Linking and validation are atomic steps on the main thread. +- The watchdog and cancellation are left out. Findings F1-F3 of `formal/parallel-loader` are not + re-modelled: loaders always finish before the main thread consumes them. +- `touchDep` assumes a language whose load pulls in other resources. PRL:131-137 exists to handle + exactly that case. +- A racy `contains` may return a false "absent". This is backed by `HashMap.resize` publishing the + empty new table before the entries are transferred, and by removals from treeified bins. +- A racy write may be lost. + +## Properties + +| ID | Statement | +|---|---| +| P1 | A rebuilt URI that is not in sources and has a binary has a complete (`fresh`) binary | +| P2 | When the build returns, every submitted store is done, failed (and logged), or dropped (and reported) | +| P3 | No thread loads a partial binary (P3partial) or an outdated binary of a URI rebuilt in this build (P3stale) | +| P4 | No unsynchronised overlapping access to the `sources` HashSet | +| P5 (pEnd) | When everything has stopped, no rebuilt URI still has an outdated binary on disk | +| Termination | The reachable graph is acyclic (Kahn) and every non-terminal state has a successor | + +## Results + +All BFS runs reach a fixpoint. The Lean theorems are in `Results.lean` and `Proof.lean`. + +| Configuration | States | P1 | P2 | P3partial | P3stale | P4 | P5 | Acyclic, no deadlock | +|---|---|---|---|---|---|---|---|---| +| Code as written, defaults (1 worker, queue 1) | 15 699 | ✗7 | ✗35 | ✗22 | ✗8 | ✗7 | ✗38 | ✓ | +| Code, 2 workers | 70 908 | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | +| Code, `ioFail` | 50 197 | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | +| Code, queue 0 (CallerRuns reachable) | 7 097 | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | ✓ | +| Code, no timeout, no `touchDep`, no race nondeterminism | 508 | ✗ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | +| Fixed (4 configurations, including 2 workers, CallerRuns, ioFail) | 105–240 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Fixed plus planted bug (removal before the write) | 162 | ✗5 | | | | | | | + +In the table, ✗n means the property is violated and the shortest counterexample has n steps. + +What is proved and what is only bounded-checked: +- **Proved for all sizes, without `sorry`** (`Proof.lean`, `fixed_safe`): P1 and P3 for the fixed + protocol. + - This covers any number of URIs (`Nat → U`), workers, clusters, executor generations and + interleavings, with the executor abstracted to per-URI lifecycle phases. + - It uses only the `propext` axiom. + - `buggy_unsafe` proves that the same statement fails once the L656 removal is added. +- **Bounded, exhaustive, via `native_decide`**: P2, P4, P5, termination and deadlock freedom, and + every counterexample. + +Sanity checks: +- The fixed model passes P1-P5, termination and deadlock freedom in 4 configurations. +- The planted bug is caught by P1 in 5 steps. +- The CallerRuns path is reachable. +- The silent CallerRuns discard is unreachable (no `execute` on a pool that is shut down). + +## Findings + +Step numbers below refer to the traces printed by `cex`. Loader bookkeeping steps are elided. + +### B1: unsynchronised concurrent mutation of the `sources` HashSet (P4). CONFIRMED + +Shortest trace, 7 steps: +1. loader1: `contains(a)` (PRL:174 → RSF:51). +2. loader1 publishes `a`. +3. main `next()` and `addResource` (MCBS:537). +4. main `execute(store a)` (MCBS:726). +5. main `sources.remove(a)` begins (MCBS:656). +6. loader1 `contains(c)` begins (RSF:51) while the removal is still in flight. + +The same set is also written by up to 4 storage workers at once (MCBS:754), and by the main thread +at MCBS:1296 while workers from the previous cluster are still removing entries. There is no +synchronisation anywhere. + +Possible consequences, reproduced in the model with `raceNondet`: +- a spurious "absent" from `contains`, so a source URI is loaded from its old binary (P3stale in + 8 steps); +- a lost `add(b)` at L1296, so `b` is not a source when loader2 loads it, and the builder relinks + a resource loaded from its binary. `addResource` (MCBS:691-705) exists precisely to avoid that. + +These corruption outcomes are plausible under the Java Memory Model and HashMap internals but were +not demonstrated in Java. The data race itself is certain. + +### B2: URI becomes binary-loadable before its binary is written (P1). CONFIRMED + +MCBS:656 removes `changedURI` from `sources` on the main thread immediately after +`storeBinaryResource` only submitted the task. MCBS:754 does the same removal properly, after +`saveResource`. From this point until the write finishes, any load of the URI takes the binary +path (RSF:51-53). The P1 trace is the first 7 steps of B1. + +Taken alone this window is harmless: with no timeout, no dependency loading in loaders and no race +nondeterminism, P1 is violated but P2, P3 and P5 hold. B3-B6 are the paths that make it +observable. + +### B3: next cluster's loaders run before storage is awaited, and read partial or outdated binaries (P3). CONFIRMED (ordering); impact depends on the language + +MCBS:668-669 starts the loader for cluster N+1. The storage await happens only afterwards, inside +`clearResourceSet` (MCBS:673 → 1204). + +Stale-read trace, 21 steps, no timeout and atomic set semantics: +1. main `next(a)`; the store of `a` goes to a new worker (726); `remove(a)` (656). +2. main `next(c)`; the store of `c` is queued (726); `remove(c)` (656). +3. main `add(b)` (1296). +4. main starts loader2 (668-669). +5. loader2 `contains(b)` returns true, so it parses `b`. +6. Loading `b` pulls in `a`: `contains(a)` returns false (removed at 656). Worker0 has not started + writing yet, so loader2 loads `a`'s **binary from the previous build** (SAR:86). + +Partial-read trace, 22 steps: the same interleaving, with worker0 already at `generateFile` +(RSF:104) when loader2 reads. + +Impact: +- A truncated binary usually throws `IOException`, and SAR:89-90 then falls back to parsing, which + mitigates the partial read. +- An outdated but complete binary is accepted silently. +- This needs a language whose load pulls in referenced resources (`touchDep`). With + `touchDep := false` the loader only ever asks about its own URI, which is still a source. + +### B4: after an await timeout, stores are silently lost and the main thread links against outdated binaries (P3stale, P5). CONFIRMED + +The default await is 1 minute with 0 retries (MCBS:804). A backlog of more than a minute makes it +fail: for example, 15 000 queue slots with 4 workers only needs about 16 ms per store. +`shutdownNow` (MCBS:879) then drops the queued stores and logs only their count. + +Their URIs were already removed from `sources` at L656, so they stay binary-loadable with the old +file on disk. Trace (P3stale on the main thread), 29 steps: +- steps 1-17 as in B3; +- 18: `shutdown` (826); +- 19: `awaitTermination` times out, then `shutdownNow` (837, 879); the store of `c` is dropped; +- 20: the executor is recreated (871); +- 21: `clearResourceSet` (1207); +- 27-29: main `next(b)`, then `resolveLazyCrossReferences` loads `a`: `contains(a)` returns false, + and the **outdated binary** of `a` is loaded (MCBS:564, SAR:86). + +P5 trace, 38 steps: the dropped store of `c` leaves `c`'s old binary on disk. Later builds install +only `toBeUpdated` and queue URIs as sources (MCBS:1524-1536), so every later build loads the +outdated binary until `c` changes again. The problem persists beyond the build. + +### B5: running stores are neither completed nor reported when the build returns (P2). CONFIRMED + +`terminateBinaryStorageExecutor` never waits after `shutdownNow`. Running tasks keep going as +zombies: they write binaries and mutate `sources`, and nothing awaits them. Meanwhile: +- the main thread clears the resource set they are serializing (MCBS:1207; the comment at MCBS:1203 + names exactly this hazard); +- the main thread starts the next cluster; +- `doUpdate` returns. + +Trace, 35 steps: B4 up to step 29; the store of `b` goes to a gen-1 worker (726); `remove(b)` +(656); the finally block's await times out and calls `shutdownNow` (684 → 879); `doUpdate` returns +while `b`'s store is still running. Only queued tasks are counted in "{} tasks not processed" +(MCBS:880). The running ones are not. + +### B6: an IOException swallowed by Xtext leaves an outdated binary that is treated as valid (P5). CONFIRMED (code path); trigger is rare + +- RSF:97-103 catches the `IOException` from `writeResource`, logs a warning and returns normally + without touching the old file. +- DLRSF:70-80 deletes storage only when an exception is thrown. +- MCBS:754 then removes the URI from `sources`. + +Trace, 45 steps, no timeout and no race: the IOException hits at step 8 for `a` and at step 23 for +`c`. At step 32 loader2 loads `a`'s outdated binary, at step 36 main does the same, and the files +stay outdated after the build. + +### Not findings + +- **Termination and deadlock freedom:** hold in every configuration (bounded). There are no cycles + and no stuck states. +- **CallerRunsPolicy:** reachable. A silent discard would need `execute` after `shutdown`, which + never happens: only the main thread submits stores, and it always recreates the executor right + after shutting it down. +- **Serialization of a detached resource after a timeout:** modelled as the store failing and + DLRSF deleting the storage, which is safe. This is a MODEL-SIMPLIFICATION: whether serialization + of a detached resource succeeds or throws was not checked in the DDK writable. diff --git a/formal/binary-storage/lean/lake-manifest.json b/formal/binary-storage/lean/lake-manifest.json new file mode 100644 index 0000000000..2b78d9f48c --- /dev/null +++ b/formal/binary-storage/lean/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.2.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "BinaryStorage", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/formal/binary-storage/lean/lakefile.toml b/formal/binary-storage/lean/lakefile.toml new file mode 100644 index 0000000000..6531b01386 --- /dev/null +++ b/formal/binary-storage/lean/lakefile.toml @@ -0,0 +1,6 @@ +name = "BinaryStorage" +version = "0.1.0" +defaultTargets = ["BinaryStorage"] + +[[lean_lib]] +name = "BinaryStorage" diff --git a/formal/binary-storage/lean/lean-toolchain b/formal/binary-storage/lean/lean-toolchain new file mode 100644 index 0000000000..acc704ffe6 --- /dev/null +++ b/formal/binary-storage/lean/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.35.0-rc2 diff --git a/formal/binary-storage/lean/theorems.txt b/formal/binary-storage/lean/theorems.txt new file mode 100644 index 0000000000..381eda2408 --- /dev/null +++ b/formal/binary-storage/lean/theorems.txt @@ -0,0 +1,20 @@ +# ; checked by formal/check.sh via #print axioms +BinaryStorage.bug_minimal native_decide +BinaryStorage.bug_p1 native_decide +BinaryStorage.bug_p2 native_decide +BinaryStorage.bug_p4 native_decide +BinaryStorage.bug_partial native_decide +BinaryStorage.bug_pEnd_iofail native_decide +BinaryStorage.bug_pEnd_timeout native_decide +BinaryStorage.bug_stale_no_timeout native_decide +BinaryStorage.bug_terminates native_decide +BinaryStorage.caller_runs_reachable_discard_unreachable native_decide +BinaryStorage.fixed_caller_runs native_decide +BinaryStorage.fixed_default native_decide +BinaryStorage.fixed_no_old_binary native_decide +BinaryStorage.fixed_two_workers_iofail native_decide +BinaryStorage.Generic.buggy_unsafe kernel +BinaryStorage.Generic.fixed_safe kernel +BinaryStorage.Generic.inv_reach kernel +BinaryStorage.Generic.inv_step kernel +BinaryStorage.planted_caught native_decide diff --git a/formal/binary-storage/tla/BinaryStorage.tla b/formal/binary-storage/tla/BinaryStorage.tla new file mode 100644 index 0000000000..1aad7faa03 --- /dev/null +++ b/formal/binary-storage/tla/BinaryStorage.tla @@ -0,0 +1,488 @@ +---------------------------- MODULE BinaryStorage ---------------------------- +(***************************************************************************) +(* Binary-model storage in MonitoredClusteringBuilderState (MCBS) and its *) +(* interaction with the cluster loop, the shared source-level URI set and *) +(* the ParallelResourceLoader (PRL) threads of the current/next cluster. *) +(* *) +(* MCBS = com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/ *) +(* xtext/builder/MonitoredClusteringBuilderState.java *) +(* PRL = .../xtext/builder/resourceloader/ParallelResourceLoader.java *) +(* *) +(* Actors: builder (main) thread; binary-storage ThreadPoolExecutor *) +(* workers, one executor per "generation" (MCBS:871 creates a new one *) +(* after every await); PRL loader jobs (one per queued URI). *) +(* *) +(* All Fix*/Plant* flags FALSE = the code as written. *) +(***************************************************************************) +EXTENDS Integers, Sequences, FiniteSets + +CONSTANTS + Clusters, \* sequence of URI sets: Clusters[1] = toBeUpdated, Clusters[k+1] = affected by cluster k + Deps, \* [URIs -> SUBSET URIs]: resources whose load may be triggered when linking/loading u + NStore, \* BINARY_STORAGE_EXECUTOR_PARALLELISM (MCBS:124) + QCap, \* BINARY_STORAGE_EXECUTOR_QUEUE_CAPACITY (MCBS:125), >= 1 + MaxTimeouts, \* bound on awaitTermination timeouts / interrupts of the builder (MCBS:837, 865) + LoaderLoadsDeps, \* may a PRL job's load of u also load u's dependencies in its local resource set? + AllowLoadFail, \* may a PRL job return an exception (LoadOperationException(uri), MCBS:595-597)? + AllowLinkFail, \* may linking throw an Exception after addResource (outer catch, MCBS:585-613)? + FixA, \* drop the main-thread sources.remove (MCBS:656); rely on MCBS:754 + FixB, \* sources set is thread-safe (every access atomic) + FixC, \* after shutdownNow, keep waiting until running store tasks have finished + FixD, \* do not store a resource whose processing threw (MCBS:654 skipped) + PlantBug \* planted: worker removes from sources BEFORE writing the binary + +URIs == UNION {Clusters[i] : i \in 1..Len(Clusters)} +NC == Len(Clusters) +Gens == 1..(NC + 1) \* one executor per await (NC-1 between clusters + 1 final) + the fresh one +WIds == Gens \X (1..NStore) +NoURI == "-" \* placeholder for URI-typed variables +NoTask == <<"-", 0>> \* placeholder for store-task variables (task = <>) +LDeps(u) == IF LoaderLoadsDeps THEN {u} \cup Deps[u] ELSE {u} \* shouldLoadFromStorage(u) is always asked for u itself + +ASSUME /\ NC >= 1 /\ NStore >= 1 /\ QCap >= 1 /\ MaxTimeouts \in Nat + /\ \A u \in URIs : Deps[u] \subseteq URIs + +VARIABLES + \* builder thread + mpc, k, queue, cur, depsTodo, mdep, mainRS, live, toAdd, qcur, gen, aret, timeouts, macc, + \* shared state + sources, \* SourceLevelURICache.getSources(): a plain java.util.HashSet (Xtext SourceLevelURICache.) + inBuild, \* URIs that are (re)built in this build (entered sources via install/queueAffected) + binary, \* on-disk binary per URI: "old" (previous build) | "none" | "partial" | "new" + \* PRL loader jobs + lpc, ltodo, lcur, lres, lacc, + \* storage executors and workers + est, eq, nstarted, wpc, wtask, wintr, wacc, + \* history + sst, \* per URI store status: no | queued | running | ok | failed | dropped | discarded + reads, \* set of <>: binary actually opened + detached \* a store serialised a resource no longer in the builder's resource set + +mainVars == <> +loadVars == <> +execVars == <> +workVars == <> +vars == <> + +Init == + /\ mpc = "head" /\ k = 1 /\ queue = Clusters[1] /\ cur = NoURI + /\ depsTodo = {} /\ mdep = NoURI /\ mainRS = {} /\ live = {} + /\ toAdd = {} /\ qcur = NoURI /\ gen = 1 /\ aret = "none" /\ timeouts = 0 /\ macc = "none" + /\ sources = Clusters[1] \* installSourceLevelURIs (MCBS:454, 1524-1536) + /\ inBuild = Clusters[1] + /\ binary \in [URIs -> {"old", "none"}] + /\ lpc = [u \in URIs |-> IF u \in Clusters[1] THEN "run" ELSE "off"] \* loadOperation.load(queue) MCBS:510 + /\ ltodo = [u \in URIs |-> IF u \in Clusters[1] THEN LDeps(u) ELSE {}] + /\ lcur = [u \in URIs |-> NoURI] /\ lres = [u \in URIs |-> "none"] /\ lacc = [u \in URIs |-> "none"] + /\ est = [g \in Gens |-> "running"] /\ eq = [g \in Gens |-> <<>>] /\ nstarted = [g \in Gens |-> 0] + /\ wpc = [w \in WIds |-> "none"] /\ wtask = [w \in WIds |-> NoTask] + /\ wintr = [w \in WIds |-> FALSE] /\ wacc = [w \in WIds |-> "none"] + /\ sst = [u \in URIs |-> "no"] /\ reads = {} /\ detached = FALSE + +----------------------------------------------------------------------------- +(* Builder thread *) + +\* Inner loop head: next() returns a finished PRL job (MCBS:531-556). +MHead == + /\ mpc = "head" + /\ IF queue = {} + THEN /\ mpc' = "endCluster" + /\ UNCHANGED <> + ELSE \E u \in queue : + /\ lpc[u] = "deliv" + /\ lpc' = [lpc EXCEPT ![u] = "done"] + /\ cur' = u + /\ queue' = queue \ {u} \* MCBS:556 / MCBS:604 + /\ IF lres[u] = "fail" + THEN \* LoadOperationException(uri): resource stays null, no store (MCBS:595-613, 654) + /\ mpc' = "rmsrc1" + /\ UNCHANGED <> + ELSE /\ mainRS' = mainRS \cup {u} \* addResource MCBS:553 + /\ live' = live \cup {<>} + /\ depsTodo' = Deps[u] + /\ mpc' = "link" + /\ UNCHANGED <> + +\* resolveLazyCrossReferences (MCBS:569): loads dependencies into the builder's resource set. +Link == + /\ mpc = "link" + /\ IF depsTodo = {} + THEN \/ /\ mpc' = "submit" /\ UNCHANGED <> + \/ /\ AllowLinkFail \* exception after addResource: + /\ live' = live \ {<>} \* resourceSet.getResources().remove(resource) MCBS:608 + /\ mainRS' = mainRS \ {cur} + /\ mpc' = IF FixD THEN "rmsrc1" ELSE "submit" \* MCBS:654 still stores it + /\ UNCHANGED <> + ELSE \E d \in depsTodo : + IF d \in mainRS + THEN /\ depsTodo' = depsTodo \ {d} + /\ UNCHANGED <> + ELSE /\ mdep' = d + /\ macc' = IF FixB THEN "none" ELSE "r" \* shouldLoadFromStorage: sources.contains(d) + /\ mpc' = "linkChk" + /\ UNCHANGED <> + /\ UNCHANGED <> + +LinkChk == + /\ mpc = "linkChk" + /\ macc' = "none" + /\ reads' = IF mdep \notin sources /\ binary[mdep] # "none" + THEN reads \cup {<<"main", mdep, binary[mdep], mdep \in inBuild>>} + ELSE reads + /\ mainRS' = mainRS \cup {mdep} + /\ depsTodo' = depsTodo \ {mdep} + /\ mpc' = "link" + /\ UNCHANGED <> + +\* storeBinaryResource (MCBS:716-736) -> ThreadPoolExecutor.execute with CallerRunsPolicy. +Submit == + /\ mpc = "submit" + /\ LET g == gen + t == <> + w == <> + IN IF est[g] # "running" + THEN \* CallerRunsPolicy.rejectedExecution: silently discards when shut down + /\ sst' = [sst EXCEPT ![cur] = "discarded"] + /\ mpc' = "rmsrc1" + /\ UNCHANGED <> + ELSE IF nstarted[g] < NStore + THEN \* workerCount < corePoolSize: addWorker(command, true) + /\ nstarted' = [nstarted EXCEPT ![g] = @ + 1] + /\ wpc' = [wpc EXCEPT ![w] = "ser"] + /\ wtask' = [wtask EXCEPT ![w] = t] + /\ sst' = [sst EXCEPT ![cur] = "running"] + /\ mpc' = "rmsrc1" + /\ UNCHANGED eq + ELSE IF Len(eq[g]) < QCap + THEN /\ eq' = [eq EXCEPT ![g] = Append(@, t)] + /\ sst' = [sst EXCEPT ![cur] = "queued"] + /\ mpc' = "rmsrc1" + /\ UNCHANGED <> + ELSE \* queue full: CallerRunsPolicy runs doStoreBinaryResource on the builder thread + /\ sst' = [sst EXCEPT ![cur] = "running"] + /\ mpc' = "crSer" + /\ UNCHANGED <> + /\ UNCHANGED <> + +\* doStoreBinaryResource on the builder thread (CallerRunsPolicy). +CrSer == + /\ mpc = "crSer" + /\ detached' = (detached \/ <> \notin live) + /\ mpc' = "crWrite" + /\ UNCHANGED <> + +CrWrite == + /\ mpc = "crWrite" + /\ \/ /\ binary' = [binary EXCEPT ![cur] = "partial"] /\ mpc' = "crWriteEnd" \* fsa.generateFile starts + \/ /\ binary' = [binary EXCEPT ![cur] = "none"] /\ mpc' = "crRm1" \* errors: deleteStorage + /\ UNCHANGED <> + +CrWriteEnd == + /\ mpc = "crWriteEnd" + /\ binary' = [binary EXCEPT ![cur] = "new"] + /\ mpc' = "crRm1" + /\ UNCHANGED <> + +CrRm1 == \* MCBS:754 + /\ mpc = "crRm1" + /\ IF FixB + THEN /\ sources' = sources \ {cur} /\ sst' = [sst EXCEPT ![cur] = "ok"] /\ mpc' = "rmsrc1" + /\ UNCHANGED macc + ELSE /\ macc' = "w" /\ mpc' = "crRm2" /\ UNCHANGED <> + /\ UNCHANGED <> + +CrRm2 == + /\ mpc = "crRm2" + /\ sources' = sources \ {cur} /\ macc' = "none" + /\ sst' = [sst EXCEPT ![cur] = "ok"] + /\ mpc' = "rmsrc1" + /\ UNCHANGED <> + +\* buildData.getSourceLevelURICache().getSources().remove(changedURI) (MCBS:656) +RmSrc1 == + /\ mpc = "rmsrc1" + /\ IF FixA + THEN /\ mpc' = "head" /\ UNCHANGED <> + ELSE IF FixB + THEN /\ sources' = sources \ {cur} /\ mpc' = "head" /\ UNCHANGED macc + ELSE /\ macc' = "w" /\ mpc' = "rmsrc2" /\ UNCHANGED sources + /\ UNCHANGED <> + +RmSrc2 == + /\ mpc = "rmsrc2" + /\ sources' = sources \ {cur} /\ macc' = "none" + /\ mpc' = "head" + /\ UNCHANGED <> + +\* Inner loop exit: loadOperation.cancel() (MCBS:661), then queueAffectedResources (MCBS:663). +EndCluster == + /\ mpc = "endCluster" + /\ IF k < NC + THEN /\ toAdd' = Clusters[k + 1] /\ mpc' = "qa" + ELSE /\ mpc' = "exitLoop" /\ UNCHANGED toAdd + /\ UNCHANGED <> + +\* queueAffectedResources: buildData.queueURI(uri); sources.add(uri) (MCBS:1296-1298, 1312-1314) +QA == + /\ mpc = "qa" + /\ IF toAdd = {} + THEN /\ mpc' = "newLoad" /\ UNCHANGED <> + ELSE \E u \in toAdd : + /\ queue' = queue \cup {u} + /\ IF FixB + THEN /\ sources' = sources \cup {u} /\ inBuild' = inBuild \cup {u} + /\ toAdd' = toAdd \ {u} /\ mpc' = "qa" /\ UNCHANGED <> + ELSE /\ qcur' = u /\ macc' = "w" /\ mpc' = "qa2" + /\ UNCHANGED <> + /\ UNCHANGED <> + +QA2 == + /\ mpc = "qa2" + /\ sources' = sources \cup {qcur} /\ inBuild' = inBuild \cup {qcur} + /\ toAdd' = toAdd \ {qcur} /\ macc' = "none" /\ mpc' = "qa" + /\ UNCHANGED <> + +\* if (!queue.isEmpty()) { loadOperation = create(...); loadOperation.load(queue); } (MCBS:667-670) +\* -- the next cluster's PRL jobs start BEFORE clearResourceSet awaits the storage executor (MCBS:672-674). +NewLoad == + /\ mpc = "newLoad" + /\ IF queue # {} + THEN /\ lpc' = [u \in URIs |-> IF u \in queue THEN "run" ELSE lpc[u]] + /\ ltodo' = [u \in URIs |-> IF u \in queue THEN LDeps(u) ELSE ltodo[u]] + /\ mpc' = "clear" + ELSE /\ mpc' = "exitLoop" /\ UNCHANGED <> + /\ UNCHANGED <> + +Terminated(g) == + /\ est[g] # "running" /\ eq[g] = <<>> + /\ \A i \in 1..NStore : wpc[<>] \in {"none", "exit"} + +\* awaitBinaryStorageExecutorTermination: from clearResourceSet (MCBS:1204) or finally (MCBS:684). +AwShut == + /\ mpc \in {"clear", "exitLoop"} + /\ est' = [est EXCEPT ![gen] = IF @ = "running" THEN "shutdown" ELSE @] \* shutdown() MCBS:826 + /\ aret' = mpc + /\ mpc' = "awWait" + /\ UNCHANGED <> + +AwOk == \* awaitTermination returns true (MCBS:837, 859) + /\ mpc = "awWait" + /\ Terminated(gen) + /\ mpc' = "awDone" + /\ UNCHANGED <> + +\* awaitTermination times out (retryCount = 0 -> loop exits) or throws InterruptedException: +\* terminateBinaryStorageExecutor -> shutdownNow() (MCBS:838-867, 877-881). +AwTimeout == + /\ mpc = "awWait" + /\ ~Terminated(gen) + /\ timeouts < MaxTimeouts + /\ timeouts' = timeouts + 1 + /\ est' = [est EXCEPT ![gen] = "stop"] + /\ sst' = [u \in URIs |-> IF \E i \in 1..Len(eq[gen]) : eq[gen][i][1] = u THEN "dropped" ELSE sst[u]] + /\ eq' = [eq EXCEPT ![gen] = <<>>] \* "{} tasks not processed" + /\ wintr' = [w \in WIds |-> IF w[1] = gen /\ wpc[w] \notin {"none", "exit"} THEN TRUE ELSE wintr[w]] + /\ mpc' = IF FixC THEN "awDrain" ELSE "awDone" + /\ UNCHANGED <> + +AwDrain == \* FixC only: wait for running tasks after shutdownNow + /\ mpc = "awDrain" + /\ Terminated(gen) + /\ mpc' = "awDone" + /\ UNCHANGED <> + +\* binaryStorageExecutor = makeBinaryStorageExecutor() (MCBS:871); then clear the resource set (MCBS:1207) +AwDone == + /\ mpc = "awDone" + /\ gen' = gen + 1 + /\ IF aret = "clear" + THEN /\ mainRS' = {} /\ live' = {} /\ k' = k + 1 /\ mpc' = "head" + ELSE /\ mpc' = "done" /\ UNCHANGED <> + /\ UNCHANGED <> + +MainStep == MHead \/ Link \/ LinkChk \/ Submit \/ CrSer \/ CrWrite \/ CrWriteEnd \/ CrRm1 \/ CrRm2 + \/ RmSrc1 \/ RmSrc2 \/ EndCluster \/ QA \/ QA2 \/ NewLoad \/ AwShut \/ AwOk \/ AwDrain \/ AwDone + +----------------------------------------------------------------------------- +(* Storage workers: doStoreBinaryResource (MCBS:746-769) *) + +WTake(w) == + LET g == w[1] IN + /\ wpc[w] = "idle" + /\ IF est[g] = "stop" \/ (est[g] = "shutdown" /\ eq[g] = <<>>) + THEN /\ wpc' = [wpc EXCEPT ![w] = "exit"] /\ UNCHANGED <> + ELSE /\ eq[g] # <<>> + /\ wtask' = [wtask EXCEPT ![w] = Head(eq[g])] + /\ eq' = [eq EXCEPT ![g] = Tail(@)] + /\ sst' = [sst EXCEPT ![Head(eq[g])[1]] = "running"] + /\ wpc' = [wpc EXCEPT ![w] = "ser"] + /\ UNCHANGED <> + +\* createResourceStorageWritable(bout).writeResource(resource): serialises the EMF resource in memory +WSer(w) == + /\ wpc[w] = "ser" + /\ detached' = (detached \/ wtask[w] \notin live) + /\ wpc' = [wpc EXCEPT ![w] = IF PlantBug THEN "rm1" ELSE "write"] + /\ UNCHANGED <> + +WWrite(w) == + LET u == wtask[w][1] IN + /\ wpc[w] = "write" + /\ \/ /\ binary' = [binary EXCEPT ![u] = "partial"] \* fsa.generateFile in progress + /\ wpc' = [wpc EXCEPT ![w] = "wend"] /\ UNCHANGED sst + \/ /\ binary' = [binary EXCEPT ![u] = "none"] \* resource has errors: deleteStorage (DLRSF:72-76) + /\ wpc' = [wpc EXCEPT ![w] = "rm1"] /\ UNCHANGED sst + /\ UNCHANGED <> + +WWriteEnd(w) == + LET u == wtask[w][1] IN + /\ wpc[w] = "wend" + /\ \/ /\ binary' = [binary EXCEPT ![u] = "new"] + /\ wpc' = [wpc EXCEPT ![w] = IF PlantBug THEN "fin" ELSE "rm1"] /\ UNCHANGED sst + \/ /\ wintr[w] \* interrupted write throws: catch deletes storage, + /\ binary' = [binary EXCEPT ![u] = "none"] \* rethrow, logged (DLRSF:77-79, MCBS:765-767) + /\ sst' = [sst EXCEPT ![u] = "failed"] + /\ wpc' = [wpc EXCEPT ![w] = "idle"] + /\ UNCHANGED <> + +WRm1(w) == \* getSources().remove(resource.getURI()) (MCBS:754) + LET u == wtask[w][1] IN + /\ wpc[w] = "rm1" + /\ IF FixB + THEN /\ sources' = sources \ {u} + /\ wpc' = [wpc EXCEPT ![w] = IF PlantBug THEN "write" ELSE "fin"] + /\ UNCHANGED wacc + ELSE /\ wacc' = [wacc EXCEPT ![w] = "w"] /\ wpc' = [wpc EXCEPT ![w] = "rm2"] /\ UNCHANGED sources + /\ UNCHANGED <> + +WRm2(w) == + LET u == wtask[w][1] IN + /\ wpc[w] = "rm2" + /\ sources' = sources \ {u} /\ wacc' = [wacc EXCEPT ![w] = "none"] + /\ wpc' = [wpc EXCEPT ![w] = IF PlantBug THEN "write" ELSE "fin"] + /\ UNCHANGED <> + +WFin(w) == + /\ wpc[w] = "fin" + /\ sst' = [sst EXCEPT ![wtask[w][1]] = "ok"] + /\ wpc' = [wpc EXCEPT ![w] = "idle"] + /\ UNCHANGED <> + +WStep(w) == WTake(w) \/ WSer(w) \/ WWrite(w) \/ WWriteEnd(w) \/ WRm1(w) \/ WRm2(w) \/ WFin(w) + +----------------------------------------------------------------------------- +(* PRL jobs: localResourceSet.getResource(uri, true) (PRL:344-349) *) +(* StorageAwareResource.load -> shouldLoadFromStorage: sources.contains(d) *) +(* via the SourceLevelURIsAdapter installed WITHOUT copy (PRL:168-174). *) + +LRun(u) == + /\ lpc[u] = "run" + /\ IF ltodo[u] = {} + THEN /\ lres' = [lres EXCEPT ![u] = "ok"] \/ (AllowLoadFail /\ lres' = [lres EXCEPT ![u] = "fail"]) + /\ lpc' = [lpc EXCEPT ![u] = "deliv"] + /\ UNCHANGED <> + ELSE \E d \in ltodo[u] : + /\ lcur' = [lcur EXCEPT ![u] = d] + /\ lacc' = [lacc EXCEPT ![u] = IF FixB THEN "none" ELSE "r"] + /\ lpc' = [lpc EXCEPT ![u] = "chk"] + /\ UNCHANGED <> + /\ UNCHANGED <> + +LChk(u) == + LET d == lcur[u] IN + /\ lpc[u] = "chk" + /\ lacc' = [lacc EXCEPT ![u] = "none"] + /\ reads' = IF d \notin sources /\ binary[d] # "none" + THEN reads \cup {<<"loader", d, binary[d], d \in inBuild>>} + ELSE reads + /\ ltodo' = [ltodo EXCEPT ![u] = @ \ {d}] + /\ lpc' = [lpc EXCEPT ![u] = "run"] + /\ UNCHANGED <> + +LStep(u) == LRun(u) \/ LChk(u) + +----------------------------------------------------------------------------- +Next == MainStep \/ AwTimeout \/ (\E w \in WIds : WStep(w)) \/ (\E u \in URIs : LStep(u)) + +Spec == Init /\ [][Next]_vars + /\ WF_vars(MainStep) + /\ \A w \in WIds : WF_vars(WStep(w)) + /\ \A u \in URIs : WF_vars(LStep(u)) + +----------------------------------------------------------------------------- +(* Properties *) + +TypeOK == + /\ sources \subseteq URIs /\ inBuild \subseteq URIs + /\ binary \in [URIs -> {"old", "none", "partial", "new"}] + /\ sst \in [URIs -> {"no", "queued", "running", "ok", "failed", "dropped", "discarded"}] + /\ gen \in Gens /\ k \in 1..NC + +\* P1: a URI rebuilt in this build is "not in sources" (= binary-loadable) only once its store has run +\* to completion (binary "new", or deleted because of errors/failure) -- not queued, not dropped, not +\* skipped, not in the middle of serialising/writing. +Writing(u) == + \/ sst[u] = "queued" + \/ \E w \in WIds : wtask[w][1] = u /\ wpc[w] \in {"ser", "write", "wend"} + \/ cur = u /\ mpc \in {"crSer", "crWrite", "crWriteEnd"} +NotSourceOnlyWhenStored == + \A u \in inBuild : u \notin sources => (sst[u] \in {"running", "ok", "failed"} /\ ~Writing(u)) + +\* P2a: at the end of the build no store is silently lost or still queued. +StoresAccounted == + mpc = "done" => \A u \in URIs : sst[u] \notin {"queued", "discarded"} +\* P2b: every submitted store eventually completes or is reported as not processed. +StoreResolved == + \A u \in URIs : (sst[u] \in {"queued", "running"}) ~> (sst[u] \in {"ok", "failed", "dropped"}) + +\* P3: PRL jobs never open a binary that is still being written. +LoaderNoPartialRead == \A r \in reads : r[1] = "loader" => r[3] # "partial" +\* P3': same for the builder thread (linking). +MainNoPartialRead == \A r \in reads : r[1] = "main" => r[3] # "partial" +\* P3'': nobody opens the previous build's binary of a resource that is rebuilt in this build. +NoStaleRead == \A r \in reads : ~(r[3] = "old" /\ r[4]) + +\* P4: the (non-thread-safe) HashSet is never mutated concurrently with any other access. +NWriters == (IF macc = "w" THEN 1 ELSE 0) + Cardinality({u \in URIs : lacc[u] = "w"}) + Cardinality({w \in WIds : wacc[w] = "w"}) +NAccesses == (IF macc # "none" THEN 1 ELSE 0) + Cardinality({u \in URIs : lacc[u] # "none"}) + Cardinality({w \in WIds : wacc[w] # "none"}) +SetThreadSafe == NWriters > 0 => NAccesses = 1 + +\* P4 refinements, to classify which overlaps exist. +NoReadDuringWrite == NWriters > 0 => \A u \in URIs : lacc[u] = "none" \* loader contains() vs a mutation +NoAddDuringRemove == ~(mpc = "qa2" /\ \E w \in WIds : wacc[w] = "w") \* main add (MCBS:1298) vs worker remove (MCBS:754) + +\* P6: never serialise a resource that was removed from / cleared out of the builder's resource set. +NoDetachedStore == ~detached + +\* P5: termination. +BuildTerminates == <>(mpc = "done") +WorkersQuiesce == <>[](\A w \in WIds : wpc[w] \in {"none", "exit", "idle"}) + +\* Witnesses (should be VIOLATED: they show that the good/interesting paths are reachable). +WitnessDone == mpc # "done" +WitnessSecondClust == k < 2 +WitnessCallerRuns == mpc # "crSer" +WitnessTimeout == timeouts = 0 +WitnessGoodRead == \A r \in reads : r[3] # "new" +WitnessStoreOk == \A u \in URIs : sst[u] # "ok" +============================================================================= diff --git a/formal/binary-storage/tla/MC.tla b/formal/binary-storage/tla/MC.tla new file mode 100644 index 0000000000..7b369ee466 --- /dev/null +++ b/formal/binary-storage/tla/MC.tla @@ -0,0 +1,16 @@ +------------------------------- MODULE MC ------------------------------- +(* Instantiation sizes for BinaryStorage (cfg files cannot write sequences/functions). *) +EXTENDS BinaryStorage +\* S: one resource per cluster; b depends on a. +ClustersS == << {"a"}, {"b"} >> +DepsS == [u \in {"a", "b"} |-> IF u = "b" THEN {"a"} ELSE {}] +\* M: two resources in cluster 1 (a2 depends on a1), b depends on both. +ClustersM == << {"a1", "a2"}, {"b"} >> +DepsM == [u \in {"a1", "a2", "b"} |-> CASE u = "a2" -> {"a1"} [] u = "b" -> {"a1", "a2"} [] OTHER -> {}] +\* L: three clusters. +ClustersL == << {"a1", "a2"}, {"b"}, {"c"} >> +DepsL == [u \in {"a1", "a2", "b", "c"} |-> CASE u = "a2" -> {"a1"} [] u = "b" -> {"a1"} [] u = "c" -> {"b"} [] OTHER -> {}] +\* R: three resources in cluster 1 so that NStore=1, QCap=1 fills the executor (CallerRunsPolicy). +ClustersR == << {"a1", "a2", "a3"}, {"b"} >> +DepsR == [u \in {"a1", "a2", "a3", "b"} |-> CASE u = "b" -> {"a1"} [] OTHER -> {}] +============================================================================= diff --git a/formal/binary-storage/tla/NOTES.md b/formal/binary-storage/tla/NOTES.md new file mode 100644 index 0000000000..0ff1c5c3ed --- /dev/null +++ b/formal/binary-storage/tla/NOTES.md @@ -0,0 +1,189 @@ +# BinaryStorage: TLA+ model notes + +This model covers binary-model storage in `MonitoredClusteringBuilderState` (MCBS): the storage executor, the shared source-level URI set, and the `ParallelResourceLoader` (PRL) jobs of the current and next cluster. Everything was checked with TLC (`formal/.tools/tla2tools.jar`). + +- MCBS: `com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java` +- PRL: `com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoader.java` +- DLRSF: `com.avaloq.tools.ddk.xtext/src/com/avaloq/tools/ddk/xtext/resource/persistence/DirectLinkingResourceStorageFacade.java` +- Xtext 2.44 classes read from the source and bytecode jars in `~/.m2`: `SourceLevelURICache`, `SourceLevelURIsAdapter`, `ResourceStorageFacade`, `StorageAwareResource`, `PortableURIs`, `AbstractResourceLoader`. + +PRL findings F1-F3 (poll timeout counter, interrupt livelock, worker leak) are known and are not modelled here. + +## Files + +| File | What | +|---|---| +| `BinaryStorage.tla` | 488 lines, about 380 non-comment. With every `Fix*`/`Plant*` flag FALSE it is the code as written. | +| `MC.tla` | Size definitions S, M, L and R. A cfg file cannot write sequences or functions. | +| `gen.sh`, `run.sh`, `check_all.sh`, `check_all.out` | Generate the cfgs, run TLC, run the whole matrix, and the results of the last run. | +| `trace.py` | Reads TLC output on stdin and prints only the variables that changed at each step. | + +Sizes (`Clusters`, `Deps`): + +| Size | Clusters | Dependencies | +|---|---|---| +| S | `<<{a},{b}>>` | b→a | +| M | `<<{a1,a2},{b}>>` | a2→a1, b→{a1,a2} | +| L | `<<{a1,a2},{b},{c}>>` | a2→a1, b→a1, c→b | +| R | `<<{a1,a2,a3},{b}>>` | b→a1. With NStore=1 and QCap=1 this fills the executor and reaches CallerRunsPolicy. | + +## What is modelled + +- **Builder thread**, one step per code step: + - `next()`/`addResource` and the queue removal (MCBS:553-556); + - linking loads dependencies into the builder's resource set, and asks `shouldLoadFromStorage`, which reads the sources set (MCBS:569); + - the outer catch for a failed load (MCBS:595-613) and for a link exception after `addResource` (MCBS:608); + - `storeBinaryResource` → `ThreadPoolExecutor.execute` (MCBS:716-736). The executor semantics are: a core thread is started while fewer than NStore exist, then the task goes into the bounded queue, and when the queue is full `CallerRunsPolicy` runs the task on the builder thread. After `shutdown`, `CallerRunsPolicy` discards the task silently. + - `sources.remove(changedURI)` (MCBS:656); + - `loadOperation.cancel()`, then `queueAffectedResources` with `queueURI` and `sources.add` (MCBS:1296-1298, 1312-1314); + - the next load operation, `load(queue)` (MCBS:667-670), which starts **before** `clearResourceSet` (MCBS:672-674); + - `awaitBinaryStorageExecutorTermination` (MCBS:819-874): `shutdown`, `awaitTermination`, then either a timeout or `InterruptedException`. With `retryCount` 0 either one leads to `shutdownNow` (MCBS:877-881), which drops the queued tasks (reported only as a count) and interrupts the workers. A new executor is then created (MCBS:871). + - clearing the resource set (MCBS:1207); the same await also runs in `finally` (MCBS:684). +- **Storage workers**: `doStoreBinaryResource` (MCBS:746-769), split into: + - serialise the resource in memory (this reads the resource and its resource set); + - `fsa.generateFile`: the binary is `partial`, then `new`. If the resource has errors, `deleteStorage` makes the binary `none` (DLRSF:72-76). An interrupted write can throw: the facade then deletes the storage and rethrows, and the error is logged (DLRSF:77-79, MCBS:765); + - `sources.remove(uri)` (MCBS:754). +- **PRL jobs**, one per queued URI: `localResourceSet.getResource(u, true)` (PRL:344-349 → AbstractResourceLoader). + - Every `StorageAwareResource.load` asks `shouldLoadFromStorage` = `!sources.contains(uri) && storageExists`. + - PRL installs the builder's live sources set into each job's resource set without copying it (PRL:168-174). + - `LoaderLoadsDeps` controls whether loading u also loads u's dependencies. PRL:131-137 unloads the extra resources such loads leave behind, so the code expects them to happen. +- **Shared sources set**: the type is `java.util.HashSet`. The bytecode shows `SourceLevelURICache.` calling `Sets.newHashSet()`. `setSourceLevelUrisWithoutCopy` wraps it in `Collections.unmodifiableSet`, which is a view, not a copy (MCBS:1536, PRL:174). Every access is modelled as two steps, begin and end. A mutation that overlaps any other thread's access is the hazard state. +- **Binary file per URI**: `old` (left by the previous build), `none`, `partial` (being written) or `new`. +- **Environment**: at most `MaxTimeouts` await timeouts or interrupts; loader load failures (`AllowLoadFail`); link exceptions (`AllowLinkFail`). +- **Fairness**: weak fairness on each thread's steps, and none on timeouts. + +**Not modelled:** the PRL result queue, counters and cancellation (covered by the earlier model); the order the sorter imposes (the model covers every order); deleted resources (`toBeDeleted`, `deleteBinaryResources`); what a corrupted HashSet actually does (only the overlap is flagged); an Error in a store. + +## Properties + +| Name | Meaning | +|---|---| +| `NotSourceOnlyWhenStored` (P1) | A URI rebuilt in this build leaves `sources` only after its store has finished, with the binary either `new` or deleted. The store must not be queued, dropped, skipped, or still serialising or writing. | +| `StoresAccounted` (P2a) | At `done`, no store is still queued and none was silently discarded. | +| `StoreResolved` (P2b, liveness) | Every queued or running store eventually ends as `ok`, `failed` (logged) or `dropped` (counted by `shutdownNow`). | +| `LoaderNoPartialRead` (P3) | PRL jobs never open a binary that is still being written. | +| `MainNoPartialRead` (P3') | Same check for the builder thread while it links. | +| `NoStaleRead` (P3'') | Nobody opens the previous build's binary of a resource that is being rebuilt in this build. | +| `SetThreadSafe` (P4) | While one thread mutates the HashSet, no other thread accesses it. Two refinements classify the overlaps: `NoReadDuringWrite` and `NoAddDuringRemove`. | +| `NoDetachedStore` (P6) | A store never serialises a resource that is no longer in the builder's resource set. This is the hazard the comment at MCBS:1203 names. | +| `BuildTerminates`, `WorkersQuiesce` (P5, liveness) | The build finishes, and all storage workers eventually stop running. | +| `Witness*` | Invariants that must be violated. They show that each good path is reachable: the build finishes, a second cluster runs, CallerRuns happens, a timeout happens, a `new` binary is read, and a store completes. | + +## Results + +The complete output is in `check_all.out`. The whole matrix took about 11 min wall time on this machine, plus 8m45s for `orig_live_M`. + +| Run | Result | +|---|---| +| The code as written, S | P1, P3, P3', P3'', P4 and P6 are **violated**. P2a, P2b and P5 pass (62,148 distinct states). | +| The code as written, M, liveness (LoadFail and LinkFail off) | P2b and P5 pass (3,859,440 distinct states, 8m45s). | +| Happy environment, M (no timeouts or failures, loaders do not load dependencies) | P1 and P4 are still **violated**. P3, P3', P3'' and P6 pass (15,188 distinct states). | +| Fixed model (FixA-D), all environment options on | All safety and liveness properties pass: S 4.5k, M 364k, M with NStore=2 561k, L 1.72M (4 min), R 911k distinct states. L with NStore=2 and 2 timeouts (2.75M) was checked for safety only. | +| Fixed model with one fix undone | Undoing A → P1 fails. B → P4 fails. C → P6 fails. D → P6 fails. So every fix is needed. | +| Planted bug (a worker removes the URI from sources before writing) | P1 is caught (154 distinct states). | +| Witnesses | All 6 are violated, as intended, in both the original and the fixed model. | + +## Findings + +Traces are the BFS-shortest ones from `./run.sh cfg/.cfg | ./trace.py`. + +### B1: The builder removes a URI from `sources` before its binary is written. CONFIRMED. Violates P1 and causes B2 and B3. + +Trace `orig_P1_nofail` (9 states), with no environment faults needed: + +1. The PRL job loads `a`, and `next()` returns it (MCBS:553-556). +2. `Link` (MCBS:569). +3. `Submit`: `storeBinaryResource` hands the task to a storage thread (MCBS:726). The task is now running or queued. +4. `RmSrc`: `getSources().remove(a)` runs **immediately** (MCBS:656). `a` is now "binary-loadable" while its store is still queued or writing. + +The worker's own `remove` after `saveResource` (MCBS:753-754) is the correct one. MCBS:656 defeats it. + +Checking this against the Java code: MCBS:654 submits the task asynchronously and MCBS:656 runs right after on the builder thread. The same line also removes URIs that were never stored, and that gives a second variant (`orig_NotSourceOnlyWhenStored`, 7 states): +- a PRL load fails, so `LoadOperationException(uri)` sets `changedURI` (MCBS:596); +- `resource` stays null, so no store happens (MCBS:654); +- the URI is still removed from `sources` (MCBS:656), and its **previous build's binary** stays on disk and becomes loadable. + +Upstream Xtext never removes URIs from the source set. It reinstalls a copy per cluster (ClusteringBuilderState.installSourceLevelURIs). + +### B2: The next cluster's loaders read a binary that is being written, or the stale one. CONFIRMED, given that loading u also loads u's dependencies. + +Trace `orig_LoaderNoPartialRead` (19 states): + +1. Cluster 1 processes `a`: `Submit` (MCBS:726), then `RmSrc` (MCBS:656), so `sources = {}`. +2. `EndCluster`, then `QA`: `b` is queued and added to `sources` (MCBS:1297-1298). +3. `NewLoad`: `loadOperation.load(queue)` (MCBS:668-669) starts b's PRL job **before** `clearResourceSet` awaits the storage executor (MCBS:673, 1204). +4. `WSer`, then `WWrite`: `fsa.generateFile` for `a` starts, so `binary[a] = partial`. +5. The PRL job loads `b`, which pulls in dependency `a`. `shouldLoadFromStorage(a)`: `a` is not in `sources` and a storage file exists, so the job opens a's **partial** binary. + +The M trace `orig_loaderpartial_M` (13 states) shows the same thing **within one cluster**: +- a1's store is writing and a1 has already been removed at MCBS:656; +- a2's PRL job, which is still running concurrently, loads dependency a1 from the partial binary. + +`orig_NoStaleRead` (17 states) is the same schedule, but the read happens before the worker starts writing. The job then loads the **previous build's** binary of `a`, even though `a` is being rebuilt. The same happens in the load-failure variant of B1, where a's old binary is never replaced. + +Checking this against the Java code: PRL:168/174 share the live set. `StorageAwareResource.load` → `ResourceStorageFacade.shouldLoadFromStorage` → `doesStorageExist`, then `getOrCreateResourceStorageLoadable` opens the file. +- A truncated read throws `IOException`, and `load` then falls back to the source (`clearAndUnload()` + `super.load`). That costs time but gives the right result. +- A `RuntimeIOException` escapes and fails the load. +- A read of the old binary is silently wrong: the resource is linked against the previous version of a resource that is being rebuilt. + +The trigger needs a PRL load of u to also load another resource. PRL:131-137 exists precisely to unload such extra resources, so this does happen in practice. How often depends on the language: derived state or inference that touches other resources during load. + +Removing MCBS:656 (FixA) closes both the partial and the stale window. Ordering the await before `load(queue)` would only close the next-cluster case, not the within-cluster one. + +### B3: After an await timeout or interrupt, the builder reads a partial binary and clears resources that are still being serialised. CONFIRMED. Violates P3' and P6. + +Trace `orig_mainpartial` (26 states): + +1. `a` is stored asynchronously and removed from `sources` (MCBS:726, 656). +2. `clearResourceSet` → await: `shutdown` (MCBS:826). `AwTimeout` covers `awaitTermination` returning false after 1 min (with `retryCount` 0 the loop exits), or an `InterruptedException`. Both lead to `shutdownNow` (MCBS:837-867, 879). The running task is **not** stopped: an interrupt does not abort in-memory serialisation. +3. `AwDone`: a new executor is created (MCBS:871) and the resource set is cleared (MCBS:1207), while the old worker is still serialising `a`. This is `orig_NoDetachedStore` (19 states, P6). +4. Cluster 2: linking `b` loads `a`. `a` is not in `sources`, and `WWrite` has just made a's binary `partial`, so the builder opens the **partial** binary. + +Checking this against the Java code: +- MCBS:1203 says "this is important as otherwise the resources would unexpectedly become detached from the resource set". The timeout path breaks exactly that guarantee. +- Serialising a detached or cleared resource reaches `PortableURIs.toPortableURI`, which calls `sourceResource.getResourceSet().getEObject(...)` (PortableURIs:183). With the resource set null or cleared, this throws a `NullPointerException` or returns null. +- DLRSF:77-79 then deletes the storage and the failure is logged at MCBS:765. That is benign. The remaining risk is a binary with non-portable references written from a half-cleared set. + +The trigger is a store batch that takes more than 1 minute, or an interrupted build thread. Separately, the catch at MCBS:865-867 swallows `InterruptedException` without re-interrupting the thread. + +### B4: Linking throws, and the resource is removed from the set but still stored. CONFIRMED, low severity. Violates P6. + +Trace `orig_NoDetachedStore` with `AllowLinkFail` (8 states): +1. An exception is thrown after `addResource`, and the outer catch runs `resourceSet.getResources().remove(resource)` (MCBS:608). +2. Execution falls through to `storeBinaryResource(resource, …)` (MCBS:654). +3. The worker serialises a resource whose `getResourceSet()` is null. + +Checking this against the Java code: `resource` is non-null on this path and nothing guards MCBS:654. The outcome is the same as the detached case in B3, most likely an NPE that is logged and the storage deleted. The practical impact is mostly log noise, plus the deletion of a binary that was valid. + +### B5: The HashSet is mutated concurrently by up to three threads without synchronisation. CONFIRMED. Violates P4, even in the happy environment. + +Traces: +- `orig_SetThreadSafe` (11 states): the builder's `remove(a)` (MCBS:656) overlaps the worker's `remove(a)` (MCBS:754). +- `orig_adr` (15 states): the builder's `sources.add(b)` in `queueAffectedResources` (MCBS:1298) overlaps the worker's `remove(a)` (MCBS:754). +- `orig_rdw` (19 states): a PRL job's `contains()`, reached through `shouldLoadFromStorage`, overlaps the worker's `remove` (MCBS:754). + +Checking this against the Java code: +- The set is a plain `HashSet` (bytecode of `SourceLevelURICache.`), shared without copying with the storage threads and every PRL job thread (MCBS:1536, PRL:168-174). +- `update` is `synchronized`, but only the builder thread takes that lock. No other lock is involved. +- Under the Java memory model these are data races. With `HashMap` they can lose an `add`: a queued URI is then missing from `sources`, and its stale binary is loaded instead of the source. That is the bad direction. They can also lose a `remove` (harmless), make `size` wrong, or let a `contains()` miss an entry during a resize. + +The model only flags the overlap. It does not simulate how the `HashMap` gets corrupted. + +### P2 (stores accounted) and P5 (termination) hold. + +- `CallerRunsPolicy` never throws. A discard after `shutdown` is unreachable, because the builder never submits between MCBS:826 and MCBS:871. So the `RejectedExecutionException` catch at MCBS:727-734 is dead code in practice. +- Tasks dropped by `shutdownNow` are reported only as a count, with no URIs. +- Orphaned running tasks finish, under fairness. + +## Minimal fixes in the fixed model + +| Fix | Change | What it resolves | +|---|---|---| +| A | Delete MCBS:656. MCBS:754 already removes the URI once the binary is saved; on the `CallerRunsPolicy` path the builder does this itself. | B1, B2 | +| B | Make the sources set thread-safe, for example by installing `ConcurrentHashMap.newKeySet()` contents via the adapter. `SourceLevelURICache` itself is Xtext-owned. Alternatively, confine all mutations to the builder thread. | B5 | +| C | After `shutdownNow`, keep waiting until the running tasks have finished before clearing or continuing. The trade-off: a truly hung store now blocks the build. | B3 | +| D | Skip `storeBinaryResource` when the outer catch ran, for example by setting `resource = null` after MCBS:608. | B4 | + +## Wall time + +About 40 min of modelling and checking (14:29 to 15:01 for the model and the matrix), plus the 8m45s liveness rerun at size M. diff --git a/formal/binary-storage/tla/cfg/ablateA.cfg b/formal/binary-storage/tla/cfg/ablateA.cfg new file mode 100644 index 0000000000..7652e190ad --- /dev/null +++ b/formal/binary-storage/tla/cfg/ablateA.cfg @@ -0,0 +1,25 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/ablateB.cfg b/formal/binary-storage/tla/cfg/ablateB.cfg new file mode 100644 index 0000000000..219261fa89 --- /dev/null +++ b/formal/binary-storage/tla/cfg/ablateB.cfg @@ -0,0 +1,25 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = FALSE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/ablateC.cfg b/formal/binary-storage/tla/cfg/ablateC.cfg new file mode 100644 index 0000000000..7811f7aa6d --- /dev/null +++ b/formal/binary-storage/tla/cfg/ablateC.cfg @@ -0,0 +1,25 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = FALSE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/ablateD.cfg b/formal/binary-storage/tla/cfg/ablateD.cfg new file mode 100644 index 0000000000..31d5344978 --- /dev/null +++ b/formal/binary-storage/tla/cfg/ablateD.cfg @@ -0,0 +1,25 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/fixed_L.cfg b/formal/binary-storage/tla/cfg/fixed_L.cfg new file mode 100644 index 0000000000..d2f0f868e7 --- /dev/null +++ b/formal/binary-storage/tla/cfg/fixed_L.cfg @@ -0,0 +1,28 @@ +CONSTANTS + Clusters <- ClustersL + Deps <- DepsL + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/fixed_L2.cfg b/formal/binary-storage/tla/cfg/fixed_L2.cfg new file mode 100644 index 0000000000..fd4ac5ab04 --- /dev/null +++ b/formal/binary-storage/tla/cfg/fixed_L2.cfg @@ -0,0 +1,25 @@ +CONSTANTS + Clusters <- ClustersL + Deps <- DepsL + NStore = 2 + QCap = 1 + MaxTimeouts = 2 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/fixed_M.cfg b/formal/binary-storage/tla/cfg/fixed_M.cfg new file mode 100644 index 0000000000..e83d08db8d --- /dev/null +++ b/formal/binary-storage/tla/cfg/fixed_M.cfg @@ -0,0 +1,28 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/fixed_M2.cfg b/formal/binary-storage/tla/cfg/fixed_M2.cfg new file mode 100644 index 0000000000..7b1d5ea1f3 --- /dev/null +++ b/formal/binary-storage/tla/cfg/fixed_M2.cfg @@ -0,0 +1,28 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/fixed_R.cfg b/formal/binary-storage/tla/cfg/fixed_R.cfg new file mode 100644 index 0000000000..4514989b57 --- /dev/null +++ b/formal/binary-storage/tla/cfg/fixed_R.cfg @@ -0,0 +1,28 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/fixed_S.cfg b/formal/binary-storage/tla/cfg/fixed_S.cfg new file mode 100644 index 0000000000..c2681d8ae6 --- /dev/null +++ b/formal/binary-storage/tla/cfg/fixed_S.cfg @@ -0,0 +1,28 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/happy_LoaderNoPartialRead.cfg b/formal/binary-storage/tla/cfg/happy_LoaderNoPartialRead.cfg new file mode 100644 index 0000000000..6015ba0aee --- /dev/null +++ b/formal/binary-storage/tla/cfg/happy_LoaderNoPartialRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT LoaderNoPartialRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/happy_MainNoPartialRead.cfg b/formal/binary-storage/tla/cfg/happy_MainNoPartialRead.cfg new file mode 100644 index 0000000000..84745f5eaf --- /dev/null +++ b/formal/binary-storage/tla/cfg/happy_MainNoPartialRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT MainNoPartialRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/happy_NoDetachedStore.cfg b/formal/binary-storage/tla/cfg/happy_NoDetachedStore.cfg new file mode 100644 index 0000000000..7ca7eed820 --- /dev/null +++ b/formal/binary-storage/tla/cfg/happy_NoDetachedStore.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/happy_NoStaleRead.cfg b/formal/binary-storage/tla/cfg/happy_NoStaleRead.cfg new file mode 100644 index 0000000000..3d3044b0ad --- /dev/null +++ b/formal/binary-storage/tla/cfg/happy_NoStaleRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoStaleRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/happy_NotSourceOnlyWhenStored.cfg b/formal/binary-storage/tla/cfg/happy_NotSourceOnlyWhenStored.cfg new file mode 100644 index 0000000000..69f1e4a913 --- /dev/null +++ b/formal/binary-storage/tla/cfg/happy_NotSourceOnlyWhenStored.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/happy_SetThreadSafe.cfg b/formal/binary-storage/tla/cfg/happy_SetThreadSafe.cfg new file mode 100644 index 0000000000..09cf51b5a4 --- /dev/null +++ b/formal/binary-storage/tla/cfg/happy_SetThreadSafe.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 2 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT SetThreadSafe +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_LoaderNoPartialRead.cfg b/formal/binary-storage/tla/cfg/orig_LoaderNoPartialRead.cfg new file mode 100644 index 0000000000..8f371a395e --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_LoaderNoPartialRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT LoaderNoPartialRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_MainNoPartialRead.cfg b/formal/binary-storage/tla/cfg/orig_MainNoPartialRead.cfg new file mode 100644 index 0000000000..c2377ba41e --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_MainNoPartialRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT MainNoPartialRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_NoDetachedStore.cfg b/formal/binary-storage/tla/cfg/orig_NoDetachedStore.cfg new file mode 100644 index 0000000000..39b7000c4e --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_NoDetachedStore.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_NoStaleRead.cfg b/formal/binary-storage/tla/cfg/orig_NoStaleRead.cfg new file mode 100644 index 0000000000..0f1c3fe3b7 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_NoStaleRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoStaleRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_NotSourceOnlyWhenStored.cfg b/formal/binary-storage/tla/cfg/orig_NotSourceOnlyWhenStored.cfg new file mode 100644 index 0000000000..11fa8a03cf --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_NotSourceOnlyWhenStored.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_P1_nofail.cfg b/formal/binary-storage/tla/cfg/orig_P1_nofail.cfg new file mode 100644 index 0000000000..aa1cf56a7f --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_P1_nofail.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_SetThreadSafe.cfg b/formal/binary-storage/tla/cfg/orig_SetThreadSafe.cfg new file mode 100644 index 0000000000..fe4ab66a42 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_SetThreadSafe.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT SetThreadSafe +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_StoresAccounted.cfg b/formal/binary-storage/tla/cfg/orig_StoresAccounted.cfg new file mode 100644 index 0000000000..818ab4fe9e --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_StoresAccounted.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT StoresAccounted +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_TypeOK.cfg b/formal/binary-storage/tla/cfg/orig_TypeOK.cfg new file mode 100644 index 0000000000..a3c7ed9036 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_TypeOK.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_adr.cfg b/formal/binary-storage/tla/cfg/orig_adr.cfg new file mode 100644 index 0000000000..b8182c28eb --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_adr.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoAddDuringRemove +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_detached_nolinkfail.cfg b/formal/binary-storage/tla/cfg/orig_detached_nolinkfail.cfg new file mode 100644 index 0000000000..96536e1d42 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_detached_nolinkfail.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_live.cfg b/formal/binary-storage/tla/cfg/orig_live.cfg new file mode 100644 index 0000000000..1d50627fde --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_live.cfg @@ -0,0 +1,20 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_live_M.cfg b/formal/binary-storage/tla/cfg/orig_live_M.cfg new file mode 100644 index 0000000000..823e84ae91 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_live_M.cfg @@ -0,0 +1,20 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +PROPERTY BuildTerminates +PROPERTY WorkersQuiesce +PROPERTY StoreResolved +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_loaderpartial_M.cfg b/formal/binary-storage/tla/cfg/orig_loaderpartial_M.cfg new file mode 100644 index 0000000000..4db789e171 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_loaderpartial_M.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersM + Deps <- DepsM + NStore = 1 + QCap = 1 + MaxTimeouts = 0 + LoaderLoadsDeps = TRUE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT LoaderNoPartialRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_mainpartial.cfg b/formal/binary-storage/tla/cfg/orig_mainpartial.cfg new file mode 100644 index 0000000000..63c1810c2a --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_mainpartial.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = FALSE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT MainNoPartialRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_rdw.cfg b/formal/binary-storage/tla/cfg/orig_rdw.cfg new file mode 100644 index 0000000000..516fda7c16 --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_rdw.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoReadDuringWrite +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/orig_stale_nofail.cfg b/formal/binary-storage/tla/cfg/orig_stale_nofail.cfg new file mode 100644 index 0000000000..ed0d2d3e7a --- /dev/null +++ b/formal/binary-storage/tla/cfg/orig_stale_nofail.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = FALSE + AllowLinkFail = FALSE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT NoStaleRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/planted.cfg b/formal/binary-storage/tla/cfg/planted.cfg new file mode 100644 index 0000000000..dfff5b9cc6 --- /dev/null +++ b/formal/binary-storage/tla/cfg/planted.cfg @@ -0,0 +1,25 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = TRUE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT TypeOK +INVARIANT NotSourceOnlyWhenStored +INVARIANT StoresAccounted +INVARIANT LoaderNoPartialRead +INVARIANT MainNoPartialRead +INVARIANT NoStaleRead +INVARIANT SetThreadSafe +INVARIANT NoDetachedStore +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/smoke.cfg b/formal/binary-storage/tla/cfg/smoke.cfg new file mode 100644 index 0000000000..ce49bf01e0 --- /dev/null +++ b/formal/binary-storage/tla/cfg/smoke.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersS + Deps <- DepsS + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessDone +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_fixed_WitnessCallerRuns.cfg b/formal/binary-storage/tla/cfg/wit_fixed_WitnessCallerRuns.cfg new file mode 100644 index 0000000000..2a8053e4f0 --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_fixed_WitnessCallerRuns.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessCallerRuns +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_fixed_WitnessDone.cfg b/formal/binary-storage/tla/cfg/wit_fixed_WitnessDone.cfg new file mode 100644 index 0000000000..6f7b34257c --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_fixed_WitnessDone.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessDone +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_fixed_WitnessGoodRead.cfg b/formal/binary-storage/tla/cfg/wit_fixed_WitnessGoodRead.cfg new file mode 100644 index 0000000000..3f36b3acfb --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_fixed_WitnessGoodRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessGoodRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_fixed_WitnessSecondClust.cfg b/formal/binary-storage/tla/cfg/wit_fixed_WitnessSecondClust.cfg new file mode 100644 index 0000000000..b3c5ab046d --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_fixed_WitnessSecondClust.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessSecondClust +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_fixed_WitnessStoreOk.cfg b/formal/binary-storage/tla/cfg/wit_fixed_WitnessStoreOk.cfg new file mode 100644 index 0000000000..6a00fd611b --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_fixed_WitnessStoreOk.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessStoreOk +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_fixed_WitnessTimeout.cfg b/formal/binary-storage/tla/cfg/wit_fixed_WitnessTimeout.cfg new file mode 100644 index 0000000000..26770ee97a --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_fixed_WitnessTimeout.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = TRUE + FixB = TRUE + FixC = TRUE + FixD = TRUE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessTimeout +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_orig_WitnessCallerRuns.cfg b/formal/binary-storage/tla/cfg/wit_orig_WitnessCallerRuns.cfg new file mode 100644 index 0000000000..13dae40e0e --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_orig_WitnessCallerRuns.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessCallerRuns +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_orig_WitnessDone.cfg b/formal/binary-storage/tla/cfg/wit_orig_WitnessDone.cfg new file mode 100644 index 0000000000..660cdec0e2 --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_orig_WitnessDone.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessDone +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_orig_WitnessGoodRead.cfg b/formal/binary-storage/tla/cfg/wit_orig_WitnessGoodRead.cfg new file mode 100644 index 0000000000..526f22d810 --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_orig_WitnessGoodRead.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessGoodRead +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_orig_WitnessSecondClust.cfg b/formal/binary-storage/tla/cfg/wit_orig_WitnessSecondClust.cfg new file mode 100644 index 0000000000..a00e0bd291 --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_orig_WitnessSecondClust.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessSecondClust +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_orig_WitnessStoreOk.cfg b/formal/binary-storage/tla/cfg/wit_orig_WitnessStoreOk.cfg new file mode 100644 index 0000000000..c481515af7 --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_orig_WitnessStoreOk.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessStoreOk +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/cfg/wit_orig_WitnessTimeout.cfg b/formal/binary-storage/tla/cfg/wit_orig_WitnessTimeout.cfg new file mode 100644 index 0000000000..5966b63913 --- /dev/null +++ b/formal/binary-storage/tla/cfg/wit_orig_WitnessTimeout.cfg @@ -0,0 +1,18 @@ +CONSTANTS + Clusters <- ClustersR + Deps <- DepsR + NStore = 1 + QCap = 1 + MaxTimeouts = 1 + LoaderLoadsDeps = TRUE + AllowLoadFail = TRUE + AllowLinkFail = TRUE + FixA = FALSE + FixB = FALSE + FixC = FALSE + FixD = FALSE + PlantBug = FALSE +SPECIFICATION Spec +INVARIANT TypeOK +INVARIANT WitnessTimeout +CHECK_DEADLOCK FALSE diff --git a/formal/binary-storage/tla/check_all.sh b/formal/binary-storage/tla/check_all.sh new file mode 100755 index 0000000000..3c37236a55 --- /dev/null +++ b/formal/binary-storage/tla/check_all.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Regenerates every configuration and runs TLC on it; prints one summary line per run. +# Runs taking a minute or more only run with FULL=1 (orig_live_M alone is ~9 min). +cd "$(dirname "$0")" +SAFE="TypeOK NotSourceOnlyWhenStored StoresAccounted LoaderNoPartialRead MainNoPartialRead NoStaleRead SetThreadSafe NoDetachedStore" +LIVE="BuildTerminates WorkersQuiesce StoreResolved" +WIT="WitnessDone WitnessSecondClust WitnessCallerRuns WitnessTimeout WitnessGoodRead WitnessStoreOk" +run() { # name size nstore qcap maxto ldeps lfail linkfail fixA fixB fixC fixD plant "inv" "props" + n=$1; shift + ./gen.sh "$n" "$@" + out=$(./run.sh "cfg/$n.cfg" 2>&1) + res=$(echo "$out" | grep -E "^Error: (Invariant|Temporal|Deadlock)|is violated" | head -1) + [ -z "$res" ] && res=$(echo "$out" | grep -q "No error has been found" && echo "OK" || echo "$out" | grep -E "Error" | head -1) + st=$(echo "$out" | grep -E "distinct states found" | tail -1 | sed -E 's/.* ([0-9]+) distinct states found.*/\1/') + t=$(echo "$out" | grep -E "^Finished in" | sed -E 's/Finished in ([^ ]+).*/\1/') + printf '%-28s %-58s %8s distinct %s\n' "$n" "$res" "$st" "$t" +} +# --- the code as written (all environment behaviours on) ----------------------------------------- +for p in $SAFE; do run "orig_$p" S 1 1 1 TRUE TRUE TRUE FALSE FALSE FALSE FALSE FALSE "$p" ""; done +run orig_live S 1 1 1 TRUE TRUE TRUE FALSE FALSE FALSE FALSE FALSE "" "$LIVE" +[ -n "${FULL:-}" ] && run orig_live_M M 1 1 1 TRUE FALSE FALSE FALSE FALSE FALSE FALSE FALSE "" "$LIVE" +# happy environment: no timeouts, no failures, loaders do not load dependencies +for p in NotSourceOnlyWhenStored LoaderNoPartialRead MainNoPartialRead NoStaleRead SetThreadSafe NoDetachedStore; do + run "happy_$p" M 2 1 0 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE "$p" ""; done +# --- fixed model: FixA..FixD, every environment behaviour on, growing sizes ---------------------- +run fixed_S S 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$SAFE" "$LIVE" +run fixed_M M 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$SAFE" "$LIVE" +[ -n "${FULL:-}" ] && run fixed_M2 M 2 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$SAFE" "$LIVE" +[ -n "${FULL:-}" ] && run fixed_L L 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$SAFE" "$LIVE" +run fixed_L2 L 2 1 2 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$SAFE" "" +[ -n "${FULL:-}" ] && run fixed_R R 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$SAFE" "$LIVE" +# --- ablations: fixed model with one fix undone ---------------------------------------------------- +run ablateA M 1 1 1 TRUE TRUE TRUE FALSE TRUE TRUE TRUE FALSE "$SAFE" "" +run ablateB M 1 1 1 TRUE TRUE TRUE TRUE FALSE TRUE TRUE FALSE "$SAFE" "" +run ablateC M 1 1 1 TRUE TRUE TRUE TRUE TRUE FALSE TRUE FALSE "$SAFE" "" +run ablateD M 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE FALSE FALSE "$SAFE" "" +# --- planted bug in the fixed model ------------------------------------------------------------------ +run planted S 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE "$SAFE" "" +# --- witnesses (each must be violated) ------------------------------------------------------------- +for w in $WIT; do run "wit_orig_$w" R 1 1 1 TRUE TRUE TRUE FALSE FALSE FALSE FALSE FALSE "$w" ""; done +for w in $WIT; do run "wit_fixed_$w" R 1 1 1 TRUE TRUE TRUE TRUE TRUE TRUE TRUE FALSE "$w" ""; done diff --git a/formal/binary-storage/tla/expected.txt b/formal/binary-storage/tla/expected.txt new file mode 100644 index 0000000000..cb325c1e95 --- /dev/null +++ b/formal/binary-storage/tla/expected.txt @@ -0,0 +1,39 @@ +orig_TypeOK OK +orig_NotSourceOnlyWhenStored VIOLATED:NotSourceOnlyWhenStored +orig_StoresAccounted OK +orig_LoaderNoPartialRead VIOLATED:LoaderNoPartialRead +orig_MainNoPartialRead VIOLATED:MainNoPartialRead +orig_NoStaleRead VIOLATED:NoStaleRead +orig_SetThreadSafe VIOLATED:SetThreadSafe +orig_NoDetachedStore VIOLATED:NoDetachedStore +orig_live OK +orig_live_M OK [full] +happy_NotSourceOnlyWhenStored VIOLATED:NotSourceOnlyWhenStored +happy_LoaderNoPartialRead OK +happy_MainNoPartialRead OK +happy_NoStaleRead OK +happy_SetThreadSafe VIOLATED:SetThreadSafe +happy_NoDetachedStore OK +fixed_S OK +fixed_M OK +fixed_M2 OK [full] +fixed_L OK [full] +fixed_L2 OK +fixed_R OK [full] +ablateA VIOLATED:NotSourceOnlyWhenStored +ablateB VIOLATED:SetThreadSafe +ablateC VIOLATED:NoDetachedStore +ablateD VIOLATED:NoDetachedStore +planted VIOLATED:NotSourceOnlyWhenStored +wit_orig_WitnessDone VIOLATED:WitnessDone +wit_orig_WitnessSecondClust VIOLATED:WitnessSecondClust +wit_orig_WitnessCallerRuns VIOLATED:WitnessCallerRuns +wit_orig_WitnessTimeout VIOLATED:WitnessTimeout +wit_orig_WitnessGoodRead VIOLATED:WitnessGoodRead +wit_orig_WitnessStoreOk VIOLATED:WitnessStoreOk +wit_fixed_WitnessDone VIOLATED:WitnessDone +wit_fixed_WitnessSecondClust VIOLATED:WitnessSecondClust +wit_fixed_WitnessCallerRuns VIOLATED:WitnessCallerRuns +wit_fixed_WitnessTimeout VIOLATED:WitnessTimeout +wit_fixed_WitnessGoodRead VIOLATED:WitnessGoodRead +wit_fixed_WitnessStoreOk VIOLATED:WitnessStoreOk diff --git a/formal/binary-storage/tla/gen.sh b/formal/binary-storage/tla/gen.sh new file mode 100755 index 0000000000..f5980e5e0a --- /dev/null +++ b/formal/binary-storage/tla/gen.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# usage: gen.sh NAME SIZE(S|M|L) NSTORE QCAP MAXTO LDEPS LFAIL LINKFAIL FIXA FIXB FIXC FIXD PLANT "INVARIANTS" "PROPERTIES" +cd "$(dirname "$0")"; mkdir -p cfg +n=$1; sz=$2 +cat > cfg/$n.cfg <> cfg/$n.cfg; done +for p in ${15}; do echo "PROPERTY $p" >> cfg/$n.cfg; done +echo "CHECK_DEADLOCK FALSE" >> cfg/$n.cfg diff --git a/formal/binary-storage/tla/run.sh b/formal/binary-storage/tla/run.sh new file mode 100755 index 0000000000..28d7e70a55 --- /dev/null +++ b/formal/binary-storage/tla/run.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# usage: ./run.sh cfg/X.cfg [extra TLC args] (checks MC.tla, i.e. BinaryStorage with the size definitions) +cd "$(dirname "$0")" +cfg=$1; shift +exec java -XX:+UseParallelGC -cp ../../.tools/tla2tools.jar tlc2.TLC -workers auto \ + -metadir "states/$(basename "$cfg" .cfg)" -config "$cfg" "$@" MC.tla diff --git a/formal/binary-storage/tla/trace.py b/formal/binary-storage/tla/trace.py new file mode 100755 index 0000000000..0fdd972c10 --- /dev/null +++ b/formal/binary-storage/tla/trace.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Condense a TLC counterexample: print each step's action and only the variables that changed.""" +import re, sys +txt = sys.stdin.read() +m = re.search(r"Error: (.*?)\n", txt) +if m: print("ERROR:", m.group(1)) +states = re.split(r"\nState (\d+): ", txt) +prev = {} +for i in range(1, len(states), 2): + n, body = states[i], states[i + 1] + body = body.split("\n\n")[0] + head, _, rest = body.partition("\n") + act = re.match(r"<(\w+)", head) + act = act.group(1) if act else head.strip() + cur = {} + for vm in re.finditer(r"^/\\ (\w+) = (.*?)(?=^/\\ |\Z)", rest, re.S | re.M): + cur[vm.group(1)] = " ".join(vm.group(2).split()) + diff = {k: v for k, v in cur.items() if prev.get(k) != v} + if n == "1": + keep = ("binary", "sources", "queue", "lpc") + diff = {k: v for k, v in cur.items() if k in keep} + print(f"{n:>3} {act:<11} " + "; ".join(f"{k}={v}" for k, v in sorted(diff.items()))) + prev = cur + if "Back to state" in body or "Stuttering" in body: + print(" ", [l for l in body.splitlines() if "Back to state" in l or "Stuttering" in l]) +m = re.search(r"(\d+) states generated, (\d+) distinct states found", txt) +if m: print("states:", m.group(1), "generated,", m.group(2), "distinct") +m = re.search(r"depth of the complete state graph search is (\d+)", txt) +if m: print("depth:", m.group(1)) diff --git a/formal/check.sh b/formal/check.sh new file mode 100755 index 0000000000..2a10bb0255 --- /dev/null +++ b/formal/check.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Local validation harness for formal/: TLC matrices vs golden verdicts, Lean builds and axioms, orphan-test check. +# Usage: formal/check.sh [--clean] [--full] [--update] [--only=|orphans] +# --clean delete .lake/, states/ and TLC metadirs first +# --full also run the expensive TLC runs (tagged [full] in expected.txt) +# --update rewrite expected.txt / theorems.txt / orphans expected.txt from this run (review the diff!) +set -u +F="$(cd "$(dirname "$0")" && pwd)" +TARGETS="parallel-loader binary-storage find-refs trie pipeline" +JAR="$F/.tools/tla2tools.jar" +JAR_SHA=936a262061c914694dfd669a543be24573c45d5aa0ff20a8b96b23d01e050e88 +LEAN_TC=leanprover/lean4:v4.35.0-rc2 +OUT="$F/.check" +FULL= CLEAN= UPDATE= ONLY= +for a in "$@"; do + case $a in + --full) FULL=1 ;; --clean) CLEAN=1 ;; --update) UPDATE=1 ;; --only=*) ONLY=${a#--only=} ;; + -h|--help) sed -n '2,7p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $a" >&2; exit 2 ;; + esac +done +case " $TARGETS orphans " in *" ${ONLY:-orphans} "*) ;; *) echo "unknown target: $ONLY" >&2; exit 2 ;; esac +[ -n "$ONLY" ] && TARGETS=${ONLY#orphans} +export FULL + +if [ -n "$CLEAN" ]; then + echo "cleaning .lake/, states/, TLC metadirs" + find "$F" -type d \( -name .lake -o -name states -o -name 'meta-*' \) -prune -exec rm -rf {} + + rm -rf "$F/find-refs/tla/meta" "$OUT" +fi +mkdir -p "$OUT" +T0=$(date +%s) +SUMMARY="$OUT/summary.txt"; : > "$SUMMARY" +result() { printf '%-28s %-4s %s\n' "$1" "$2" "$3" >> "$SUMMARY"; } + +# ---------------------------------------------------------------- TLA+ +# Maps a TLC log (stdin) to OK | VIOLATED: | DEADLOCK | ERROR; temporal violations are unnamed by TLC. +VERDICT=' +function verdict(s) { + if (match(s, /Invariant [A-Za-z0-9_]+ is violated/)) return "VIOLATED:" substr(s, RSTART + 10, RLENGTH - 22) + if (match(s, /Evaluating invariant [A-Za-z0-9_]+ failed/)) return "VIOLATED:" substr(s, RSTART + 21, RLENGTH - 28) + if (s ~ /Temporal properties were violated/) return "VIOLATED:TEMPORAL" + if (s ~ /Deadlock reached/) return "DEADLOCK" + if (s ~ /No error has been found/) return "OK" + return "ERROR" +}' +classify() { # name logfile + if [ -f "$2" ]; then awk -v n="$1" "$VERDICT"' { b = b $0 "\n" } END { print n, verdict(b) }' "$2"; else echo "$1 ERROR"; fi +} + +tla_matrix() { # target -> normalised " " lines on stdout + local d="$F/$1/tla" + case $1 in + parallel-loader) sh "$d/check_all.sh" | awk "$VERDICT"' NF { print $1 ":" $2, verdict($0) }' ;; + binary-storage) sh "$d/check_all.sh" | awk "$VERDICT"' NF { print $1, ($2 == "OK" ? "OK" : verdict($0)) }' ;; + find-refs) bash "$d/matrix.sh" | sed -n 's/^== \([^ ]*\) .*/\1/p' | while read -r n; do classify "$n" "$d/runs/$n.out"; done ;; + trie) sh "$d/runall.sh" | awk '{ print $2 }' | while read -r n; do classify "$n" "$d/out/$n.log"; done ;; + pipeline) bash "$d/run_all.sh" ${FULL:+--live} | awk '{ print $1 }' | while read -r n; do classify "$n" "$d/logs/$n.log"; done ;; + esac +} + +tla_ok=1 +if ! command -v java >/dev/null; then + echo "java not found"; tla_ok= +elif [ ! -f "$JAR" ]; then + echo "missing $JAR; fetch it with:" + echo " gh release download v1.7.4 -R tlaplus/tlaplus -p tla2tools.jar -D formal/.tools" + echo " expected SHA-256 $JAR_SHA"; tla_ok= +else + sha=$( (shasum -a 256 "$JAR" 2>/dev/null || sha256sum "$JAR") | cut -d' ' -f1) + [ "$sha" = "$JAR_SHA" ] || { echo "tla2tools.jar SHA-256 mismatch: $sha (expected $JAR_SHA)"; tla_ok=; } +fi + +tla_check() { # target + local t=$1 gold="$F/$1/tla/expected.txt" got="$OUT/$1.tla.txt" s=$(date +%s) + [ -n "$tla_ok" ] || { result "$t/tla" FAIL "TLC unavailable"; return; } + echo "== $t/tla" + tla_matrix "$t" > "$got" + if [ -n "$UPDATE" ]; then + # keep the [full] tags of runs that were skipped this time + awk 'NR == FNR { seen[$1] = 1; print; next } $3 == "[full]" && !($1 in seen)' "$got" "$gold" 2>/dev/null > "$got.new" + [ -n "$FULL" ] && [ -f "$gold" ] && awk 'NR == FNR { if ($3 == "[full]") full[$1] = 1; next } { print $0 (($1 in full) ? " [full]" : "") }' "$gold" "$got.new" > "$got.tag" && mv "$got.tag" "$got.new" + mv "$got.new" "$gold" + fi + local want="$OUT/$t.tla.want" + if [ -n "$FULL" ]; then awk 'NF && $1 !~ /^#/ { print $1, $2 }' "$gold"; else awk 'NF && $1 !~ /^#/ && $3 != "[full]" { print $1, $2 }' "$gold"; fi 2>/dev/null | sort > "$want" + if diff <(sort "$got") "$want" > "$OUT/$t.tla.diff"; then + result "$t/tla" PASS "$(wc -l < "$got" | tr -d ' ') runs, $(( $(date +%s) - s ))s" + else + sed 's/^/ /' "$OUT/$t.tla.diff" + result "$t/tla" FAIL "golden mismatch (< got, > expected), $(( $(date +%s) - s ))s" + fi +} + +# ---------------------------------------------------------------- Lean +# Lists every user-written theorem of the package (declaration ranges exclude generated lemmas). +lean_probe() { # libs... -> Lean source on stdout + echo "import Lean.Elab.Command" + for l in "$@"; do echo "import $l"; done + printf 'open Lean in\nrun_cmd do\n let env ← getEnv\n let roots : List Name := [%s]\n' "$(printf '`%s, ' "$@" | sed 's/, $//')" + cat <<'EOF' + for (n, ci) in env.constants.map₁.toList do + if let .thmInfo _ := ci then + unless n.isInternal do + if let some idx := env.getModuleIdxFor? n then + if roots.contains env.header.moduleNames[idx.toNat]!.getRoot then + if (← findDeclarationRanges? n).isSome then IO.println s!"THM {n}" +EOF +} + +lean_check() { # target; prints details to stdout, one summary via result() + local t=$1 d="$F/$1/lean" log="$OUT/$1.lean.log" s=$(date +%s) fail= + local tc; tc=$(tr -d '[:space:]' < "$d/lean-toolchain") + [ "$tc" = "$LEAN_TC" ] || { result "$t/lean" FAIL "toolchain $tc, want $LEAN_TC"; return; } + elan toolchain list 2>/dev/null | grep -q "^$tc" || { result "$t/lean" FAIL "$tc not installed (no downloads)"; return; } + (cd "$d" && lake build) > "$log" 2>&1 || { tail -20 "$log"; result "$t/lean" FAIL "lake build failed ($log)"; return; } + # sorry/admit outside comments, or reported by the elaborator + local holes; holes=$(fd -0 -e lean -E .lake . "$d" | xargs -0 perl -0777 -ne 's{/-.*?-/}{}gs; s{--[^\n]*}{}g; print "$ARGV\n" if /\b(sorry|admit)\b/') + grep -q "declaration uses 'sorry'" "$log" && holes="$holes (build log)" + [ -n "$holes" ] && { echo "sorry/admit in: $holes"; fail=1; } + local libs; libs=$(awk '/^\[/ { sec = $0 } sec == "[[lean_lib]]" && /^name *=/ { gsub(/^name *= *"|"$/, ""); print }' "$d/lakefile.toml") + local list="$d/theorems.txt" probe="$d/.lake/check_axioms.lean" pout="$OUT/$t.axioms.txt" + lean_probe $libs > "$probe" + (cd "$d" && lake env lean "$probe") 2>&1 | awk '/^THM / { print $2 }' | sort > "$OUT/$t.all" + # #print axioms for every listed and every declared theorem + { lean_probe $libs; { [ -f "$list" ] && awk 'NF && $1 !~ /^#/ { print $1 }' "$list"; cat "$OUT/$t.all"; } | sort -u | sed 's/^/#print axioms /'; } > "$probe" + (cd "$d" && lake env lean "$probe") > "$pout" 2>&1 + # one line per theorem: + awk ' + function flush() { if (n == "") return + k = "kernel"; split(ax, a, /[][, ]+/) + for (i in a) if (a[i] != "" && a[i] !~ /^(propext|Classical\.choice|Quot\.sound)$/) { + if (a[i] ~ /^(Lean\.ofReduceBool|Lean\.trustCompiler)$/ || a[i] ~ /\._native\.native_decide\.ax_[0-9_]+$/) { if (k == "kernel") k = "native_decide" } + else { k = "BAD:" a[i]; break } } + print n, k; n = "" } + /^THM / { next } + /^'\''/ { flush(); n = $0; sub(/^'\''/, "", n); sub(/'\''.*/, "", n); ax = $0; sub(/^[^:]*:/, "", ax) + if ($0 ~ /does not depend on any axioms/) ax = ""; next } + n != "" { ax = ax " " $0 } + END { flush() }' "$pout" | sort > "$OUT/$t.got" + [ -n "$UPDATE" ] && { echo "# ; checked by formal/check.sh via #print axioms"; cat "$OUT/$t.got"; } > "$list" + grep -Eq ':[0-9]+:[0-9]+: error' "$pout" && { grep -E ': error' "$pout" | head -5; fail=1; } + awk 'NF && $1 !~ /^#/ { print $1, $2 }' "$list" | sort > "$OUT/$t.want" + if ! diff <(awk '{ print $1 }' "$OUT/$t.want") "$OUT/$t.all" > /dev/null; then + echo "theorem list drift (< theorems.txt, > declared):"; diff <(awk '{ print $1 }' "$OUT/$t.want") "$OUT/$t.all" | grep '^[<>]'; fail=1 + fi + if ! diff "$OUT/$t.got" "$OUT/$t.want" > "$OUT/$t.lean.diff"; then + echo "axiom class mismatch (< actual, > theorems.txt):"; grep '^[<>]' "$OUT/$t.lean.diff"; fail=1 + fi + awk '{ printf " %-66s %s\n", $1, $2 }' "$OUT/$t.got" + local nk nn; nk=$(grep -c ' kernel$' "$OUT/$t.got"); nn=$(grep -c ' native_decide$' "$OUT/$t.got") + result "$t/lean" "$([ -n "$fail" ] && echo FAIL || echo PASS)" "$nk kernel + $nn native_decide theorems, $(( $(date +%s) - s ))s" +} + +# ---------------------------------------------------------------- orphaned tests +orphans_check() { # runs on a snapshot of HEAD so uncommitted work in the tree does not change the verdict + local o="$F/readonly/orphans" snap got="$OUT/orphans.txt" s=$(date +%s) + snap=$(mktemp -d "${TMPDIR:-/tmp}/formal-check.XXXXXX") + git -C "$(git -C "$F" rev-parse --show-toplevel)" archive HEAD | tar -x -C "$snap" + "$o/check-test-reachability.sh" "$snap" > "$OUT/orphans.log" 2>&1; local rc=$? + rm -rf "$snap" + { echo "exit $rc"; awk -F'\t' '$1 ~ /UNREACHABLE$/ { print "UNREACHABLE", $3 }' "$OUT/orphans.log" | sort; } > "$got" + "$o/selftest.sh" > "$OUT/orphans-selftest.log" 2>&1 && echo "selftest exit 0" >> "$got" || echo "selftest exit $?" >> "$got" + [ -n "$UPDATE" ] && cp "$got" "$o/expected.txt" + if diff "$got" "$o/expected.txt" > "$OUT/orphans.diff"; then + result readonly/orphans PASS "exit $rc as expected, $(grep -c ^UNREACHABLE "$got") unreachable, $(( $(date +%s) - s ))s" + else + sed 's/^/ /' "$OUT/orphans.diff"; result readonly/orphans FAIL "outcome differs from expected.txt" + fi +} + +# Lean builds run in the background while TLC runs in the foreground. +( for t in $TARGETS; do lean_check "$t" > "$OUT/$t.lean.report" 2>&1; done ) & +LEAN_PID=$! +for t in $TARGETS; do tla_check "$t"; done +[ -z "$ONLY" ] || [ "$ONLY" = orphans ] && orphans_check +wait $LEAN_PID +for t in $TARGETS; do echo "== $t/lean"; cat "$OUT/$t.lean.report"; done + +echo; echo "== summary ($(( $(date +%s) - T0 ))s${FULL:+, --full})" +sort "$SUMMARY" +! grep -q ' FAIL ' "$SUMMARY" diff --git a/formal/f1-fix.patch b/formal/f1-fix.patch new file mode 100644 index 0000000000..440736a3a3 --- /dev/null +++ b/formal/f1-fix.patch @@ -0,0 +1,41 @@ +diff --git a/com.avaloq.tools.ddk.xtext.builder/META-INF/MANIFEST.MF b/com.avaloq.tools.ddk.xtext.builder/META-INF/MANIFEST.MF +index ce944bfc6..7447b5500 100644 +--- a/com.avaloq.tools.ddk.xtext.builder/META-INF/MANIFEST.MF ++++ b/com.avaloq.tools.ddk.xtext.builder/META-INF/MANIFEST.MF +@@ -2,7 +2,7 @@ Manifest-Version: 1.0 + Bundle-ManifestVersion: 2 + Bundle-Name: com.avaloq.tools.ddk.xtext.builder + Bundle-SymbolicName: com.avaloq.tools.ddk.xtext.builder;singleton:=true +-Bundle-Version: 17.3.1.qualifier ++Bundle-Version: 17.3.2.qualifier + Bundle-Vendor: Avaloq Group AG + Require-Bundle: org.eclipse.xtext.builder, + org.eclipse.xtext.ui, +diff --git a/com.avaloq.tools.ddk.xtext.builder/pom.xml b/com.avaloq.tools.ddk.xtext.builder/pom.xml +index e5d12cee1..710f52ccb 100644 +--- a/com.avaloq.tools.ddk.xtext.builder/pom.xml ++++ b/com.avaloq.tools.ddk.xtext.builder/pom.xml +@@ -6,7 +6,7 @@ + 18.0.1-SNAPSHOT + ../ddk-parent + +- 17.3.1-SNAPSHOT ++ 17.3.2-SNAPSHOT + com.avaloq.tools.ddk + com.avaloq.tools.ddk.xtext.builder + eclipse-plugin +diff --git a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoader.java b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoader.java +index 26e570a79..86cf48c22 100644 +--- a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoader.java ++++ b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/resourceloader/ParallelResourceLoader.java +@@ -218,7 +218,9 @@ public class ParallelResourceLoader extends AbstractResourceLoader { + Triple result = null; + try { + result = resourceQueue.poll(waitTime, TimeUnit.MILLISECONDS); +- toProcess--; ++ if (result != null) { ++ toProcess--; ++ } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } diff --git a/formal/find-refs/fix-cme.patch b/formal/find-refs/fix-cme.patch new file mode 100644 index 0000000000..031e9d3ad1 --- /dev/null +++ b/formal/find-refs/fix-cme.patch @@ -0,0 +1,39 @@ +diff --git a/com.avaloq.tools.ddk.xtext.ui/META-INF/MANIFEST.MF b/com.avaloq.tools.ddk.xtext.ui/META-INF/MANIFEST.MF +index 609ac5245..79e450c5d 100644 +--- a/com.avaloq.tools.ddk.xtext.ui/META-INF/MANIFEST.MF ++++ b/com.avaloq.tools.ddk.xtext.ui/META-INF/MANIFEST.MF +@@ -2,7 +2,7 @@ Manifest-Version: 1.0 + Bundle-ManifestVersion: 2 + Bundle-Name: com.avaloq.tools.ddk.xtext.ui + Bundle-SymbolicName: com.avaloq.tools.ddk.xtext.ui;singleton:=true +-Bundle-Version: 17.3.3.qualifier ++Bundle-Version: 17.3.4.qualifier + Bundle-Vendor: Avaloq Group AG + Bundle-RequiredExecutionEnvironment: JavaSE-21 + Bundle-ActivationPolicy: lazy +diff --git a/com.avaloq.tools.ddk.xtext.ui/pom.xml b/com.avaloq.tools.ddk.xtext.ui/pom.xml +index c0672c774..46bec8091 100644 +--- a/com.avaloq.tools.ddk.xtext.ui/pom.xml ++++ b/com.avaloq.tools.ddk.xtext.ui/pom.xml +@@ -6,7 +6,7 @@ + 18.0.1-SNAPSHOT + ../ddk-parent + +- 17.3.3-SNAPSHOT ++ 17.3.4-SNAPSHOT + com.avaloq.tools.ddk + com.avaloq.tools.ddk.xtext.ui + eclipse-plugin +diff --git a/com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProvider.java b/com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProvider.java +index 657f03292..f6a8a46cd 100644 +--- a/com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProvider.java ++++ b/com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProvider.java +@@ -115,7 +115,7 @@ public class FastReferenceSearchResultContentProvider extends ReferenceSearchRes + if (newInput instanceof ReferenceSearchResult && v instanceof TreeViewer) { + ((ReferenceSearchResult) newInput).addListener(this); + this.viewer = (TreeViewer) v; +- for (IReferenceDescription referenceDescription : ((ReferenceSearchResult) newInput).getMatchingReferences()) { ++ for (IReferenceDescription referenceDescription : Lists.newArrayList(((ReferenceSearchResult) newInput).getMatchingReferences())) { + addReference(referenceDescription); + } + } diff --git a/formal/find-refs/lean/.gitignore b/formal/find-refs/lean/.gitignore new file mode 100644 index 0000000000..01f8cdb637 --- /dev/null +++ b/formal/find-refs/lean/.gitignore @@ -0,0 +1 @@ +.lake/ diff --git a/formal/find-refs/lean/FindRefs.lean b/formal/find-refs/lean/FindRefs.lean new file mode 100644 index 0000000000..48fc047933 --- /dev/null +++ b/formal/find-refs/lean/FindRefs.lean @@ -0,0 +1,6 @@ +import FindRefs.State +import FindRefs.Buggy +import FindRefs.Fixed +import FindRefs.Check +import FindRefs.Proof +import FindRefs.Theorems diff --git a/formal/find-refs/lean/FindRefs/Buggy.lean b/formal/find-refs/lean/FindRefs/Buggy.lean new file mode 100644 index 0000000000..d8801514ca --- /dev/null +++ b/formal/find-refs/lean/FindRefs/Buggy.lean @@ -0,0 +1,127 @@ +/- +Faithful model of FastReferenceSearchResultContentProvider as written (FRS = that file). +Every shared-memory access that is not protected by a common lock is its own step. +-/ +import FindRefs.State + +namespace FindRefs.Buggy +open FindRefs + +/-- Individual repairs, applied one at a time to expose bugs masked by earlier ones. + `Patch.none` is the code as written. -/ +structure Patch where + lost : Bool := false -- batch-add/flag test-and-set and isEmpty/flag-clear under the batch lock + order : Bool := false -- inputChanged: removeListener(old) before rootNodes.clear() + snap : Bool := false -- inputChanged: iterate a snapshot copy of matchingReferences + async : Bool := false -- Reset via asyncExec instead of syncExec (does not wait) +deriving Repr + +def Patch.none : Patch := {} + +/-- Search-thread steps. -/ +def sStep (P : Patch) (c : Cfg) (s : St) : List (String × St) := + match s.spc with + | .idle => searchStarts c s + | .fire r => + -- RSR.fireEvent: synchronized(listeners) { for l : listeners ... } + if s.listening then [(s!"S RSR.fireEvent(Added ref{r}) takes listeners lock -> FRS:170", { s with lockS := true, spc := .get r })] + else [(s!"S RSR.fireEvent(Added ref{r}): FRS not a listener, dropped", { s with spc := .idle })] + | .get r => + let u := uriOfRef s r + match getRoot s.roots u with + | some n => [(s!"S FRS:158 rootNodes.get(uri{u}) = node{n}", { s with spc := .child r n })] + | none => [(s!"S FRS:158 rootNodes.get(uri{u}) = null", { s with spc := .put r })] + | .put r => + let u := uriOfRef s r + let n := s.nodes.length + [(s!"S FRS:160-161 rootNodes.put(uri{u}, node{n})", + { s with nodes := s.nodes ++ [(u, epochOfRef s r)], roots := setRoot s.roots u (some n), spc := .bat r n })] + | .bat r n => [(s!"S FRS:162-164 batchAddNodes.add(node{n})", { s with batch := s.batch ++ [n], spc := .child r n })] + | .child r n => [(s!"S FRS:137 attach ref{r} under node{n}", { s with attach := s.attach ++ [(r, n)], spc := .flg })] + | .flg => + if P.lost then + if s.flag then [("S [patched] synchronized(batch){flag already true}; release listeners lock", { s with spc := .idle, lockS := false })] + else [("S [patched] synchronized(batch){flag := true; schedule}; release listeners lock", { s with flag := true, jobs := s.jobs + 1, spc := .idle, lockS := false })] + else if s.flag then [("S FRS:173 isUIUpdateScheduled == true -> skip schedule; release listeners lock", { s with spc := .idle, lockS := false })] + else [("S FRS:173 isUIUpdateScheduled == false", { s with spc := .setf })] + | .setf => [("S FRS:174 isUIUpdateScheduled = true", { s with flag := true, spc := .sched })] + | .sched => [("S FRS:175 new UIUpdater().schedule(); release listeners lock", { s with jobs := s.jobs + 1, spc := .idle, lockS := false })] + | .rfire => + if s.listening then + if P.async then [("S [patched] RSR.fireEvent(Reset) -> Display.asyncExec (no wait)", { s with syncReq := true, spc := .idle })] + else [("S RSR.fireEvent(Reset) takes listeners lock -> FRS:178 Display.syncExec (blocks)", { s with lockS := true, syncReq := true, spc := .rwait })] + else [("S RSR.fireEvent(Reset): FRS not a listener, dropped", { s with spc := .idle, resetPending := false })] + | .rwait => + if s.syncReq then [] else [("S FRS:178 syncExec returns; release listeners lock", { s with lockS := false, spc := .idle })] + +/-- UI-thread steps. -/ +def uStep (P : Patch) (_c : Cfg) (s : St) : List (String × St) := + match s.upc with + | .idle => + (if s.jobs > 0 then + [(s!"UI UIUpdater starts: FRS:207-210 snapshot+clear batch {s.batch}; FRS:212-215 viewer.add each", + { s with jobs := s.jobs - 1, batch := [], viewer := s.batch.foldr sins s.viewer, upc := .uRefresh })] + else []) ++ + (if s.syncReq then + [("UI Reset runnable FRS:181-186: viewer.remove(rootNodes.values), rootNodes.clear, refresh", + { s with roots := clearRoots s.roots, viewer := [], syncReq := false, resetPending := false })] + else []) ++ + (if s.switchB > 0 && s.input == .A then + if P.order then + [("UI [patched] user shows other result B: inputChanged(A,B) removeListener first", + { s with switchB := s.switchB - 1, upc := .wRemove })] + else + [("UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear", + { s with switchB := s.switchB - 1, roots := clearRoots s.roots, upc := .wRemove })] + else []) ++ + (if s.switchB > 0 && s.input == .B then + [("UI user shows result A again: setInput(A) -> inputChanged(B,A) FRS:111 rootNodes.clear", + { s with switchB := s.switchB - 1, roots := clearRoots s.roots, upc := .bAdd })] + else []) + | .uRefresh => [("UI FRS:216 viewer.refresh() (root items := rootNodes.values)", { s with viewer := sset (rootsVals s.roots), upc := .uCheck })] + | .uCheck => + if P.lost then + if s.batch.isEmpty then [("UI [patched] synchronized(batch){empty -> flag := false}", { s with flag := false, upc := .idle })] + else [("UI [patched] synchronized(batch){non-empty -> schedule(250)}", { s with jobs := s.jobs + 1, upc := .idle })] + else [(s!"UI FRS:217 batchAddNodes.isEmpty() = {s.batch.isEmpty}", { s with upc := .uDecide s.batch.isEmpty })] + | .uDecide e => + if e then [("UI FRS:220 isUIUpdateScheduled = false", { s with flag := false, upc := .idle })] + else [("UI FRS:218 schedule(250)", { s with jobs := s.jobs + 1, upc := .idle })] + | .wRemove => + -- RSR.removeListener is synchronized(listeners): blocks while S holds it + if s.lockS then [] + else if P.order then [("UI [patched] A.removeListener(this); rootNodes.clear", { s with listening := false, roots := clearRoots s.roots, upc := .wFinish })] + else [("UI FRS:113 A.removeListener(this)", { s with listening := false, upc := .wFinish })] + | .wFinish => [("UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh", { s with input := .B, viewer := sset (rootsVals s.roots), upc := .idle })] + | .bAdd => + if s.lockS then [] + else [("UI FRS:116 A.addListener(this)", { s with listening := true, upc := .bIter })] + | .bIter => + if P.snap then [("UI [patched] snapshot = copy of matchingReferences", { s with snap := some s.matching, upc := .bLoop 0 s.mc })] + else [("UI FRS:118 matchingReferences.iterator()", { s with upc := .bLoop 0 s.mc })] + | .bLoop cur m => + let l := s.snap.getD s.matching + if cur == l.length then [("UI FRS:118 iterator.hasNext() = false", { s with snap := none, upc := .bDone })] + else if s.snap.isNone && s.mc != m then [("UI FRS:118 iterator.next() throws ConcurrentModificationException", { s with cme := true, upc := .idle })] + else + let r := (l[cur]?).getD 0 + [(s!"UI FRS:118 next() = ref{r}", { s with upc := .bGet cur m r })] + | .bGet cur m r => + let u := uriOfRef s r + match getRoot s.roots u with + | some n => [(s!"UI FRS:119->158 rootNodes.get(uri{u}) = node{n}", { s with upc := .bChild cur m r n })] + | none => [(s!"UI FRS:119->158 rootNodes.get(uri{u}) = null", { s with upc := .bPut cur m r })] + | .bPut cur m r => + let u := uriOfRef s r + let n := s.nodes.length + [(s!"UI FRS:119->160-161 rootNodes.put(uri{u}, node{n})", + { s with nodes := s.nodes ++ [(u, epochOfRef s r)], roots := setRoot s.roots u (some n), upc := .bBat cur m r n })] + | .bBat cur m r n => [(s!"UI FRS:119->162-164 batchAddNodes.add(node{n})", { s with batch := s.batch ++ [n], upc := .bChild cur m r n })] + | .bChild cur m r n => [(s!"UI FRS:119->137 attach ref{r} under node{n}", { s with attach := s.attach ++ [(r, n)], upc := .bLoop (cur + 1) m })] + | .bDone => [("UI ContentViewer.setInput(A): input := A; refresh", { s with input := .A, viewer := sset (rootsVals s.roots), resetPending := s.resetPending && resetInFlight s, upc := .idle })] + | .uDecideF | .wFix | .bFix => [] + +def next (P : Patch) (c : Cfg) (s : St) : List (String × St) := + if s.cme then [] else sStep P c s ++ uStep P c s + +end FindRefs.Buggy diff --git a/formal/find-refs/lean/FindRefs/Check.lean b/formal/find-refs/lean/FindRefs/Check.lean new file mode 100644 index 0000000000..027767c4d8 --- /dev/null +++ b/formal/find-refs/lean/FindRefs/Check.lean @@ -0,0 +1,106 @@ +/- +Exhaustive BFS to a fixpoint, properties, and shortest-counterexample extraction. +-/ +import FindRefs.State + +namespace FindRefs +open Std + +abbrev Next := St → List (String × St) + +structure Explored where + order : Array St -- BFS discovery order + parent : HashMap St (Option (String × St)) -- predecessor + step label + edges : Nat + complete : Bool -- fixpoint reached within the cap + +/-- Breadth-first exploration until no new states (or `cap` states). -/ +def explore (next : Next) (init : St) (cap : Nat := 2000000) : Explored := Id.run do + let mut parent : HashMap St (Option (String × St)) := HashMap.emptyWithCapacity 4096 + parent := parent.insert init none + let mut order : Array St := #[init] + let mut i := 0 + let mut edges := 0 + -- `order` doubles as the BFS queue; loop bounded by `cap` + for _ in [0:cap] do + if h : i < order.size then + let s := order[i] + i := i + 1 + for (lbl, t) in next s do + edges := edges + 1 + if !parent.contains t then + parent := parent.insert t (some (lbl, s)) + order := order.push t + else + break + return { order, parent, edges, complete := i ≥ order.size } + +def trace (e : Explored) (s : St) : List String := Id.run do + let mut acc : List String := [] + let mut cur := s + for _ in [0:10000] do + match e.parent[cur]? with + | some (some (lbl, p)) => acc := lbl :: acc; cur := p + | _ => break + return acc + +/-! Properties -/ + +def uiIdle (s : St) : Bool := s.upc == .idle + +/-- No internal step pending: only environment actions (new accept/reset, user switches) remain. -/ +def quiescent (s : St) : Bool := + s.spc == .idle && s.upc == .idle && s.jobs == 0 && !s.syncReq && s.input == .A && s.listening && !s.cme + +/-- P1 no lost update: at quiescence every matching reference hangs under a shown root node. -/ +def p1 (s : St) : Bool := + !quiescent s || s.matching.all fun r => s.viewer.any fun n => s.attach.contains (r, n) + +/-- P1' (root-level form): at quiescence every current root node is shown. -/ +def p1root (s : St) : Bool := + !quiescent s || (rootsVals s.roots).all fun n => s.viewer.contains n + +/-- P2 one root node per URI: between UI runnables, no two shown root nodes share a URI. -/ +def p2 (s : St) : Bool := + !uiIdle s || (s.viewer.map (uriOfNode s)).Nodup + +/-- P3 no stale node: between UI runnables, once the viewer reflects the last reset, + every shown node belongs to the current input and the current search. -/ +def p3 (s : St) : Bool := + !uiIdle s || + (match s.input with + | .B => s.viewer.isEmpty + | .A => s.resetPending || s.viewer.all fun n => epochOfNode s n == s.epoch) + +/-- P4 no exception on the UI thread. -/ +def p4 (s : St) : Bool := !s.cme + +def props : List (String × (St → Bool)) := + [("P1 no lost update (refs)", p1), ("P1' no lost update (roots)", p1root), + ("P2 one root per URI", p2), ("P3 no stale node", p3), ("P4 no UI-thread CME", p4)] + +/-- Deadlock: no successor although some actor is still mid-flight. -/ +def deadlock (next : Next) (s : St) : Bool := + (next s).isEmpty && !s.cme && + !(s.spc == .idle && s.upc == .idle && s.jobs == 0 && !s.syncReq) + +def firstViolation (e : Explored) (p : St → Bool) : Option St := + e.order.find? fun s => !p s + +def allProps (next : Next) (e : Explored) : Bool := + e.complete && props.all (fun (_, p) => e.order.all p) && e.order.all (fun s => !deadlock next s) + +def report (name : String) (next : Next) (init : St) : IO Unit := do + let e := explore next init + IO.println s!"== {name}: {e.order.size} states, {e.edges} edges, fixpoint={e.complete}" + let checks := props ++ [("DL no deadlock", fun s => !deadlock next s)] + for (pn, p) in checks do + match firstViolation e p with + | none => IO.println s!" {pn}: holds" + | some s => + let t := trace e s + IO.println s!" {pn}: VIOLATED, shortest trace ({t.length} steps):" + for (l, k) in t.zipIdx do IO.println s!" {k+1}. {l}" + IO.println s!" final: roots={s.roots} batch={s.batch} flag={s.flag} jobs={s.jobs} viewer={s.viewer} matching={s.matching} attach={s.attach} nodes={s.nodes} epoch={s.epoch} input={repr s.input} spc={repr s.spc} upc={repr s.upc} lockS={s.lockS} syncReq={s.syncReq}" + +end FindRefs diff --git a/formal/find-refs/lean/FindRefs/Fixed.lean b/formal/find-refs/lean/FindRefs/Fixed.lean new file mode 100644 index 0000000000..5f77bf2e80 --- /dev/null +++ b/formal/find-refs/lean/FindRefs/Fixed.lean @@ -0,0 +1,82 @@ +/- +Repaired design (not the Java as written). Changes w.r.t. FRS: + F1 addReference/resourceNode + flag test-and-set run atomically under one provider lock + (synchronized(batchAddNodes)); the UIUpdater's "batch empty? clear flag : reschedule" + is atomic under the same lock. [fixes lost update] + F2 Reset is handled on the search thread under that lock (clear rootNodes and batch, + ensure an updater is scheduled); no Display.syncExec, so no UI wait while the + RSR listeners monitor is held. [fixes deadlock, stale batch] + F3 inputChanged removes the old listener before clearing, and rebuilds from a + snapshot taken after addListener, under the provider lock; addReference skips a + reference already attached to the current root. [fixes stale-on-switch, CME, + duplicate root / lost ref] +`plant := true` plants an obvious bug (flag set without scheduling the job). +-/ +import FindRefs.State + +namespace FindRefs.Fixed +open FindRefs + +/-- resourceNode + attach child, deduplicated (part of the atomic addReference). -/ +def attachRef (s : St) (r : Nat) (withBatch : Bool) : St := + let u := uriOfRef s r + match getRoot s.roots u with + | some n => if s.attach.contains (r, n) then s else { s with attach := s.attach ++ [(r, n)] } + | none => + let n := s.nodes.length + { s with nodes := s.nodes ++ [(u, epochOfRef s r)], roots := setRoot s.roots u (some n), + batch := if withBatch then s.batch ++ [n] else s.batch, attach := s.attach ++ [(r, n)] } + +/-- Flag test-and-set under the batch lock. -/ +def ensureScheduled (plant : Bool) (s : St) : St := + if s.flag then s else { s with flag := true, jobs := if plant then s.jobs else s.jobs + 1 } + +/-- Atomic addReference under the provider lock (F1 + dedupe). -/ +def addRef (plant : Bool) (s : St) (r : Nat) : St := ensureScheduled plant (attachRef s r true) + +/-- Rebuild from a snapshot of matchingReferences (F3); the viewer is refreshed right after. -/ +def rebuild (s : St) : St := + s.matching.foldl (fun acc r => attachRef acc r false) { s with roots := clearRoots s.roots } + +def sStep (plant : Bool) (c : Cfg) (s : St) : List (String × St) := + match s.spc with + | .idle => searchStarts c s + | .fire r => + if s.listening then [(s!"S fireEvent(Added ref{r}) -> atomic addReference", { addRef plant s r with spc := .idle })] + else [(s!"S fireEvent(Added ref{r}): not a listener, dropped", { s with spc := .idle })] + | .rfire => + if s.listening then + [("S fireEvent(Reset) -> atomic clear rootNodes+batch, ensure updater", + ensureScheduled false { s with roots := clearRoots s.roots, batch := [], spc := .idle })] + else [("S fireEvent(Reset): not a listener, dropped", { s with spc := .idle, resetPending := false })] + | _ => [] + +def uStep (_c : Cfg) (s : St) : List (String × St) := + match s.upc with + | .idle => + (if s.jobs > 0 then + [("UI UIUpdater: atomic snapshot+clear batch, add, refresh", + { s with jobs := s.jobs - 1, batch := [], viewer := rootsVals s.roots, + resetPending := s.resetPending && resetInFlight s, upc := .uDecideF })] + else []) ++ + (if s.switchB > 0 && s.input == .A then + [("UI inputChanged(A,B): A.removeListener", { s with switchB := s.switchB - 1, listening := false, upc := .wFix })] + else []) ++ + (if s.switchB > 0 && s.input == .B then + [("UI inputChanged(B,A): A.addListener", { s with switchB := s.switchB - 1, listening := true, upc := .bFix })] + else []) + | .uDecideF => + if s.batch.isEmpty then [("UI atomic: batch empty -> flag := false", { s with flag := false, upc := .idle })] + else [("UI atomic: batch non-empty -> schedule(250)", { s with jobs := s.jobs + 1, upc := .idle })] + | .wFix => [("UI inputChanged(A,B): clear rootNodes+batch; setInput(B) refresh", + { s with roots := clearRoots s.roots, batch := [], input := .B, viewer := [], upc := .idle })] + | .bFix => [("UI inputChanged(B,A): rebuild from snapshot; setInput(A) refresh", + let s1 := rebuild s + { s1 with input := .A, viewer := rootsVals s1.roots, + resetPending := s.resetPending && resetInFlight s, upc := .idle })] + | _ => [] + +def next (plant : Bool) (c : Cfg) (s : St) : List (String × St) := + sStep plant c s ++ uStep c s + +end FindRefs.Fixed diff --git a/formal/find-refs/lean/FindRefs/Proof.lean b/formal/find-refs/lean/FindRefs/Proof.lean new file mode 100644 index 0000000000..46190f01c0 --- /dev/null +++ b/formal/find-refs/lean/FindRefs/Proof.lean @@ -0,0 +1,203 @@ +/- +All-sizes proof (any Cfg: any number of URIs, accepts, resets, switches) for the fixed model: +"no lost root node": whenever no UIUpdater is scheduled or running, every root node in +rootNodes is shown in the viewer. Proved via an inductive invariant. +-/ +import FindRefs.Fixed + +namespace FindRefs.Proof +open FindRefs + +inductive Reach (next : St → List (String × St)) (init : St) : St → Prop + | init : Reach next init init + | step {s t : St} {l : String} : Reach next init s → (l, t) ∈ next s → Reach next init t + +def Inv (s : St) : Prop := + (s.batch ≠ [] → s.flag = true) ∧ + (s.flag = true → s.jobs > 0 ∨ s.upc = .uDecideF) ∧ + (∀ n ∈ rootsVals s.roots, n ∈ s.viewer ∨ n ∈ s.batch) + +theorem rootsVals_set (l : List (Option Nat)) (u n m : Nat) : + m ∈ rootsVals (setRoot l u (some n)) → m = n ∨ m ∈ rootsVals l := by + unfold rootsVals setRoot + simp only [List.mem_filterMap, id_eq, exists_eq_right] + intro h + rcases List.mem_or_eq_of_mem_set h with h | h + · exact Or.inr h + · exact Or.inl (Option.some.inj h) + +theorem rootsVals_clear (l : List (Option Nat)) : rootsVals (clearRoots l) = [] := by + unfold rootsVals clearRoots + induction l with + | nil => rfl + | cons x xs ih => simp + +theorem init_inv (c : Cfg) : Inv (St.init c) := by + refine ⟨?_, ?_, ?_⟩ + · simp [St.init] + · simp [St.init] + · intro n hn + have : rootsVals (List.replicate c.nU (none : Option Nat)) = [] := by + unfold rootsVals; induction c.nU <;> simp_all [List.replicate_succ] + simp [St.init, this] at hn + +/-- searchStarts only touches search-side fields. -/ +theorem searchStarts_frame (c : Cfg) (s : St) (p : String × St) (h : p ∈ searchStarts c s) : + p.2.batch = s.batch ∧ p.2.flag = s.flag ∧ p.2.jobs = s.jobs ∧ p.2.upc = s.upc ∧ + p.2.roots = s.roots ∧ p.2.viewer = s.viewer := by + unfold searchStarts at h + simp only [List.mem_append] at h + rcases h with h | h + · split at h + · simp only [List.mem_map, List.mem_range] at h + obtain ⟨_, _, rfl⟩ := h + simp + · simp at h + · split at h + · simp only [List.mem_cons, List.not_mem_nil, or_false] at h + subst h; simp + · simp at h + +theorem attachRef_frame (s : St) (r : Nat) (wb : Bool) : + (Fixed.attachRef s r wb).flag = s.flag ∧ (Fixed.attachRef s r wb).jobs = s.jobs ∧ + (Fixed.attachRef s r wb).upc = s.upc ∧ (Fixed.attachRef s r wb).viewer = s.viewer ∧ + (wb = false → (Fixed.attachRef s r wb).batch = s.batch) := by + unfold Fixed.attachRef + cases h : getRoot s.roots (uriOfRef s r) with + | none => cases wb <;> simp [h] + | some n => simp only [h]; split <;> simp + +theorem attachRef_cover (s : St) (r : Nat) + (h3 : ∀ n ∈ rootsVals s.roots, n ∈ s.viewer ∨ n ∈ s.batch) : + ∀ n ∈ rootsVals (Fixed.attachRef s r true).roots, + n ∈ (Fixed.attachRef s r true).viewer ∨ n ∈ (Fixed.attachRef s r true).batch := by + unfold Fixed.attachRef + cases h : getRoot s.roots (uriOfRef s r) with + | some n => simp only [h]; split <;> exact h3 + | none => + simp only [h, ite_true] + intro n hn + rcases rootsVals_set _ _ _ _ hn with h | h + · exact Or.inr (by simp [h]) + · rcases h3 n h with h' | h' + · exact Or.inl h' + · exact Or.inr (by simp [h']) + +theorem ensureScheduled_inv (s : St) + (h2 : s.flag = true → s.jobs > 0 ∨ s.upc = .uDecideF) + (h3 : ∀ n ∈ rootsVals s.roots, n ∈ s.viewer ∨ n ∈ s.batch) : + Inv (Fixed.ensureScheduled false s) := by + unfold Fixed.ensureScheduled + split + · rename_i hf; exact ⟨fun _ => hf, h2, h3⟩ + · exact ⟨fun _ => rfl, fun _ => Or.inl (by simp), h3⟩ + +theorem addRef_inv (s : St) (r : Nat) (hI : Inv s) : Inv (Fixed.addRef false s r) := by + obtain ⟨_, h2, h3⟩ := hI + obtain ⟨ef, ej, eu, _, _⟩ := attachRef_frame s r true + unfold Fixed.addRef + apply ensureScheduled_inv + · intro hf; rw [ej, eu]; exact h2 (ef ▸ hf) + · exact attachRef_cover s r h3 + +theorem rebuild_frame (s : St) : + (Fixed.rebuild s).batch = s.batch ∧ (Fixed.rebuild s).flag = s.flag ∧ + (Fixed.rebuild s).jobs = s.jobs ∧ (Fixed.rebuild s).upc = s.upc := by + unfold Fixed.rebuild + suffices ∀ (l : List Nat) (acc : St), acc.batch = s.batch → acc.flag = s.flag → acc.jobs = s.jobs → + acc.upc = s.upc → + let t := l.foldl (fun acc r => Fixed.attachRef acc r false) acc + t.batch = s.batch ∧ t.flag = s.flag ∧ t.jobs = s.jobs ∧ t.upc = s.upc from + this _ _ rfl rfl rfl rfl + intro l + induction l with + | nil => intro acc h1 h2 h3 h4; exact ⟨h1, h2, h3, h4⟩ + | cons r rs ih => + intro acc h1 h2 h3 h4 + obtain ⟨ef, ej, eu, _, eb⟩ := attachRef_frame acc r false + exact ih _ (by rw [eb rfl]; exact h1) (by rw [ef]; exact h2) (by rw [ej]; exact h3) (by rw [eu]; exact h4) + +theorem step_inv (c : Cfg) (s : St) (hI : Inv s) (p : String × St) (hp : p ∈ Fixed.next false c s) : + Inv p.2 := by + obtain ⟨h1, h2, h3⟩ := hI + unfold Fixed.next at hp + rcases List.mem_append.mp hp with hp | hp + · -- search thread + unfold Fixed.sStep at hp + split at hp + · obtain ⟨eb, ef, ej, eu, er, ev⟩ := searchStarts_frame c s p hp + exact ⟨by rw [eb, ef]; exact h1, by rw [ef, ej, eu]; exact h2, by rw [er, ev, eb]; exact h3⟩ + · split at hp + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + exact addRef_inv s _ ⟨h1, h2, h3⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + exact ⟨h1, h2, h3⟩ + · split at hp + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + apply ensureScheduled_inv + · exact h2 + · intro n hn; simp [rootsVals_clear] at hn + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + exact ⟨h1, h2, h3⟩ + · simp at hp + · -- UI thread + unfold Fixed.uStep at hp + split at hp + · simp only [List.mem_append] at hp + rcases hp with (hp | hp) | hp + · split at hp + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + refine ⟨fun h => absurd rfl h, fun _ => Or.inr rfl, fun n hn => Or.inl hn⟩ + · simp at hp + · split at hp + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + refine ⟨h1, fun hf => Or.inl ((h2 hf).resolve_right (by intro h; simp_all)), h3⟩ + · simp at hp + · split at hp + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + refine ⟨h1, fun hf => Or.inl ((h2 hf).resolve_right (by intro h; simp_all)), h3⟩ + · simp at hp + · split at hp + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + refine ⟨fun h => absurd (List.isEmpty_iff.mp (by assumption)) h, fun h => by simp at h, h3⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + exact ⟨h1, fun _ => Or.inl (by simp), h3⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + refine ⟨fun h => absurd rfl h, fun hf => Or.inl ((h2 hf).resolve_right (by intro h; simp_all)), + fun n hn => by simp [rootsVals_clear] at hn⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false] at hp; subst hp + obtain ⟨eb, ef, ej, _⟩ := rebuild_frame s + refine ⟨fun h => ?_, fun hf => ?_, fun n hn => Or.inl hn⟩ + · simp only [eb, ef] at h ⊢; exact h1 h + · simp only [ef, ej] at hf ⊢ + exact Or.inl ((h2 hf).resolve_right (by intro h; simp_all)) + · simp at hp + +theorem reach_inv (c : Cfg) (s : St) (h : Reach (Fixed.next false c) (St.init c) s) : Inv s := by + induction h with + | init => exact init_inv c + | step _ hmem ih => exact step_inv c _ ih _ hmem + +/-- Main theorem (all sizes): in the fixed design, whenever no UIUpdater is scheduled or + running, every root node in rootNodes is shown in the viewer. -/ +theorem no_lost_root (c : Cfg) (s : St) (h : Reach (Fixed.next false c) (St.init c) s) + (hj : s.jobs = 0) (hu : s.upc = .idle) : ∀ n ∈ rootsVals s.roots, n ∈ s.viewer := by + obtain ⟨h1, h2, h3⟩ := reach_inv c s h + have hf : s.flag = false := by + cases hfl : s.flag + · rfl + · rcases h2 hfl with h | h + · omega + · rw [hu] at h; cases h + have hb : s.batch = [] := by + cases hb : s.batch with + | nil => rfl + | cons x xs => + have := h1 (by rw [hb]; simp) + rw [this] at hf; cases hf + intro n hn + rcases h3 n hn with h | h + · exact h + · rw [hb] at h; cases h + +end FindRefs.Proof diff --git a/formal/find-refs/lean/FindRefs/State.lean b/formal/find-refs/lean/FindRefs/State.lean new file mode 100644 index 0000000000..c4676698aa --- /dev/null +++ b/formal/find-refs/lean/FindRefs/State.lean @@ -0,0 +1,152 @@ +/- +Shared state of the find-references model. + +Code under study (all line numbers refer to this file unless prefixed): + com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/ + FastReferenceSearchResultContentProvider.java ("FRS:") +Xtext 2.44 org.eclipse.xtext.ui.editor.findrefs.ReferenceSearchResult ("RSR.") + +Actors + * S : the search job thread (Eclipse InternalSearchJob -> ReferenceQuery.run). + It calls RSR.reset() once at start and RSR.accept(ref) per match. + RSR.accept/reset mutate `matchingReferences` WITHOUT a lock and then call + RSR.fireEvent, which holds `synchronized(listeners)` while calling + FRS.searchResultChanged. => all event deliveries are serialised by the + `listeners` monitor (modelled by `lockS`). + * UI : the single SWT display thread. Each UI runnable (a UIJob body, a + syncExec runnable, a viewer.setInput -> inputChanged call) runs to + completion without interleaving with other UI runnables, but S steps + interleave freely between its individual shared-memory accesses. + * Jobs: `new UIUpdater().schedule()` / `schedule(250)` put a UIJob in the + job queue (`jobs` = number of scheduled-but-not-started instances). + A UIJob runs later as a UI runnable. + * Viewer: JFace TreeViewer. `refresh()` re-reads getElements() = rootNodes.values(), + so after refresh the root items are exactly the current map values. + `add` of an already present element is deduplicated (AbstractTreeViewer + itemExists). Children are fetched lazily => a reference is visible + iff it is attached to a root node that is shown (optimistic). + * Input: the viewer input is either the result A under study, or some other + result B (a different reference search shown in the same page). +-/ +import Std.Data.HashSet +import Std.Data.HashMap + +namespace FindRefs + +structure Cfg where + nU : Nat := 2 -- number of resource URIs + adds : Nat := 3 -- accept() calls available to S + resets : Nat := 1 -- reset() calls available to S (search (re)start) + switches : Nat := 2 -- page input switches A->B / B->A available to the user +deriving Repr + +/-- Search-thread program counter. `r` = reference id, `n` = node id. -/ +inductive SPc where + | idle + | fire (r : Nat) -- RSR.accept: matchingReferences.add done; fireEvent next + | get (r : Nat) -- FRS:158 rootNodes.get(uri) + | put (r : Nat) -- FRS:160-161 new node, rootNodes.put + | bat (r n : Nat) -- FRS:162-164 synchronized batchAddNodes.add + | child (r n : Nat) -- FRS:137 new DynamicReferenceSearchViewTreeNode(resourceNode, ..) + | flg -- FRS:173 read isUIUpdateScheduled + | setf -- FRS:174 isUIUpdateScheduled = true + | sched -- FRS:175 new UIUpdater().schedule(); then release `listeners` + | rfire -- RSR.reset: matchingReferences.clear done; fireEvent next + | rwait -- FRS:178 blocked in Display.syncExec +deriving BEq, Hashable, Repr, DecidableEq + +/-- UI-thread program counter (which UI runnable is mid-flight, and where). -/ +inductive UPc where + | idle + | uRefresh -- UIUpdater: FRS:216 viewer.refresh() next + | uCheck -- FRS:217 batchAddNodes.isEmpty() next + | uDecide (empty : Bool) -- FRS:217-221 reschedule or clear flag next + | uDecideF -- fixed model: atomic decide under batch lock + | wRemove -- inputChanged(A,B): cleared (FRS:111); removeListener(A) next (FRS:113) + | wFinish -- inputChanged(A,B): addListener(B) + setInput refresh + | bAdd -- inputChanged(B,A): cleared; removeListener(B), addListener(A) next (FRS:116) + | bIter -- FRS:118 getMatchingReferences().iterator() next + | bLoop (c m : Nat) -- FRS:118 for-each over live matchingReferences (cursor c, expected modCount m) + | bGet (c m r : Nat) -- FRS:119 -> 158 + | bPut (c m r : Nat) -- FRS:119 -> 160-161 + | bBat (c m r n : Nat) -- FRS:119 -> 162-164 + | bChild (c m r n : Nat) -- FRS:119 -> 137 + | bDone -- ContentViewer.setInput: input := A, refresh + | wFix -- fixed model: second half of inputChanged(A,B) + | bFix -- fixed model: second half of inputChanged(B,A) +deriving BEq, Hashable, Repr, DecidableEq + +inductive Inp where + | A | B +deriving BEq, Hashable, Repr, DecidableEq + +structure St where + spc : SPc := .idle + addB : Nat + resetB : Nat + switchB : Nat + lockS : Bool := false -- S holds RSR(A).listeners monitor + listening : Bool := true -- FRS registered as listener of A + matching : List Nat := [] -- RSR(A).matchingReferences + mc : Nat := 0 -- its ArrayList modCount + epoch : Nat := 0 -- number of reset() calls so far (ghost) + refs : List (Nat × Nat) := [] -- ref id ↦ (uri, epoch) (ghost) + nodes : List (Nat × Nat) := [] -- node id ↦ (uri, epoch of the ref that created it) (ghost) + roots : List (Option Nat) -- FRS.rootNodes : uri ↦ node + batch : List Nat := [] -- FRS.batchAddNodes + flag : Bool := false -- FRS.isUIUpdateScheduled + attach : List (Nat × Nat) := [] -- (ref, node): reference node attached under root node + jobs : Nat := 0 -- scheduled, not yet started UIUpdater instances + upc : UPc := .idle + syncReq : Bool := false -- Reset runnable posted by syncExec, not yet run + input : Inp := .A + viewer : List Nat := [] -- root items shown (sorted, dedup) + resetPending : Bool := false -- ghost: reset() called, effect not yet reflected in viewer + snap : Option (List Nat) := none -- UI-local snapshot (only used by the `snap` patch) + cme : Bool := false -- ConcurrentModificationException thrown on UI thread +deriving BEq, Hashable, Repr + +def St.init (c : Cfg) : St := + { addB := c.adds, resetB := c.resets, switchB := c.switches, roots := List.replicate c.nU none } + +/-! Helpers -/ + +def rootsVals (roots : List (Option Nat)) : List Nat := roots.filterMap id + +def getRoot (roots : List (Option Nat)) (u : Nat) : Option Nat := (roots[u]?).getD none + +def setRoot (roots : List (Option Nat)) (u : Nat) (v : Option Nat) : List (Option Nat) := roots.set u v + +def clearRoots (roots : List (Option Nat)) : List (Option Nat) := roots.map (fun _ => none) + +def sins (x : Nat) : List Nat → List Nat + | [] => [x] + | y :: ys => if x < y then x :: y :: ys else if x = y then y :: ys else y :: sins x ys + +def sset (l : List Nat) : List Nat := l.foldr sins [] + +def uriOfRef (s : St) (r : Nat) : Nat := ((s.refs[r]?).getD (0, 0)).1 +def epochOfRef (s : St) (r : Nat) : Nat := ((s.refs[r]?).getD (0, 0)).2 +def uriOfNode (s : St) (n : Nat) : Nat := ((s.nodes[n]?).getD (0, 0)).1 +def epochOfNode (s : St) (n : Nat) : Nat := ((s.nodes[n]?).getD (0, 0)).2 + +/-- Ghost bookkeeping for P3: a refresh only "reflects the last reset" when no Reset + event is still in flight to the provider. -/ +def resetInFlight (s : St) : Bool := s.spc == .rfire || s.spc == .rwait || s.syncReq + +/-- S idle: environment chooses the next RSR call (accept of a ref in uri u, or reset). -/ +def searchStarts (c : Cfg) (s : St) : List (String × St) := + (if s.addB > 0 then + (List.range c.nU).map fun u => + let r := s.refs.length + (s!"S RSR.accept(ref{r} in uri{u}): matchingReferences.add", + { s with addB := s.addB - 1, refs := s.refs ++ [(u, s.epoch)], matching := s.matching ++ [r], + mc := s.mc + 1, spc := .fire r }) + else []) ++ + (if s.resetB > 0 then + [("S RSR.reset(): matchingReferences.clear [search (re)started]", + { s with resetB := s.resetB - 1, matching := [], mc := s.mc + 1, epoch := s.epoch + 1, + resetPending := true, spc := .rfire })] + else []) + +end FindRefs diff --git a/formal/find-refs/lean/FindRefs/Theorems.lean b/formal/find-refs/lean/FindRefs/Theorems.lean new file mode 100644 index 0000000000..33dac3d44a --- /dev/null +++ b/formal/find-refs/lean/FindRefs/Theorems.lean @@ -0,0 +1,36 @@ +/- +Bounded model-checking results as kernel-accepted theorems (native_decide over the +exhaustive BFS to a fixpoint). Default Cfg: 2 URIs, 3 accepts, 1 reset, 2 input switches. +-/ +import FindRefs.Buggy +import FindRefs.Fixed +import FindRefs.Check +import FindRefs.Proof + +namespace FindRefs.Theorems +open FindRefs + +def cfg : Cfg := {} + +/-- Verdict vector for a model: fixpoint reached, then for P1, P1', P2, P3, P4, DL whether it holds. -/ +def verdicts (next : St → List (String × St)) (c : Cfg) : List Bool := + let e := explore next (St.init c) + e.complete :: (props.map fun (_, p) => (firstViolation e p).isNone) ++ + [(firstViolation e (fun s => !deadlock next s)).isNone] + +/-- As written: P1, P1', P3, P4 and deadlock-freedom are violated; P2 holds. -/ +theorem buggy_verdicts : verdicts (Buggy.next .none cfg) cfg = [true, false, false, true, false, false, false] := by + native_decide + +/-- Fixed design: everything holds within the bound. -/ +theorem fixed_verdicts : verdicts (Fixed.next false cfg) cfg = [true, true, true, true, true, true, true] := by + native_decide + +/-- Sanity: a planted bug (flag set without scheduling) is caught (P1, P1'). -/ +theorem planted_caught : verdicts (Fixed.next true cfg) cfg = [true, false, false, true, true, true, true] := by + native_decide + +end FindRefs.Theorems + +#print axioms FindRefs.Proof.no_lost_root +#print axioms FindRefs.Theorems.fixed_verdicts diff --git a/formal/find-refs/lean/NOTES.md b/formal/find-refs/lean/NOTES.md new file mode 100644 index 0000000000..63c07cbeeb --- /dev/null +++ b/formal/find-refs/lean/NOTES.md @@ -0,0 +1,188 @@ +# Find-references batching: Lean 4 model + +Subject: `com.avaloq.tools.ddk.xtext.ui/src/com/avaloq/tools/ddk/xtext/ui/editor/findrefs/FastReferenceSearchResultContentProvider.java` +(called **FRS** below; `FRS:n` is a line number). Context read: Xtext 2.44 `ReferenceSearchResult` (RSR), +`ReferenceQuery`, `ReferenceSearchViewPage`, superclass `ReferenceSearchResultContentProvider`; +Eclipse Search 3.19 `InternalSearchUI`, `SearchView`, `SearchViewManager`; JFace 3.40 `AbstractTreeViewer`. + +Toolchain: `leanprover/lean4:v4.35.0-rc2`, no Mathlib, no dependencies, nothing downloaded. + +``` +lake build # models, BFS checker, all-sizes proof, native_decide theorems (~65 s) +lake env lean Report.lean # every variant with shortest traces (~2 min 15 s); output saved in report.txt +``` + +| File | Lines | Contents | +|---|---|---| +| `FindRefs/State.lean` | 152 | state, pcs, helpers, environment (accept/reset choices) | +| `FindRefs/Buggy.lean` | 127 | the code as written, plus a patch ladder (`Patch`) | +| `FindRefs/Fixed.lean` | 82 | repaired design, plus `plant` (a deliberately planted bug) | +| `FindRefs/Check.lean` | 106 | BFS to a fixpoint, properties, shortest-trace extraction | +| `FindRefs/Proof.lean` | 203 | all-sizes inductive-invariant proof (no `sorry`) | +| `FindRefs/Theorems.lean` | 36 | `native_decide` verdict theorems + `#print axioms` | +| total | 723 | (plus `Report.lean`, 11) | + +## Threading assumptions + +* **Search thread S.** Eclipse `InternalSearchJob` runs `ReferenceQuery.run`, which calls `RSR.reset()` once and then `RSR.accept(ref)` once per match. There is one S per result: `InternalSearchUI.runSearchInBackground` refuses to start a query that is already running. + * `accept` and `reset` change `matchingReferences` (an `ArrayList`) with no lock. They then call `fireEvent`, which holds `synchronized(listeners)` while it calls `FRS.searchResultChanged`. + * So events reach FRS one at a time, and S holds the `listeners` monitor for the whole handler (`lockS`). +* **UI thread.** UI runnables (a UIJob body, a `syncExec` runnable, `setInput`→`inputChanged`) never interleave with each other. S steps can interleave between any two shared-memory accesses inside a UI runnable. + * `RSR.addListener` and `RSR.removeListener` are `synchronized(listeners)`, so the UI blocks on them while S is inside a handler. +* **UIJobs.** `new UIUpdater().schedule()` and `schedule(250)` add one pending instance (`jobs`), which runs later as a UI runnable. +* **Viewer (JFace).** `refresh()` rebuilds the root items from `getElements()` = `rootNodes.values()`. `add` of an element that is already present does nothing (`itemExists`). Children are fetched lazily, so a reference counts as visible if its root node is shown. This is an optimistic assumption. +* **Input switches (user).** The search page, viewer and FRS are one per view. `setInput(B)` for another reference-search result B calls `inputChanged(A,B)`, and showing A again calls `inputChanged(B,A)`. `page.setInput(null)` does not touch the viewer (`ReferenceSearchViewPage.setInput`), so it is not modelled. B is static (no search runs on it). +* **Memory model.** Sequential consistency (optimistic): the unsynchronized `batchAddNodes.isEmpty()` at FRS:217 and the `ArrayList` children are treated as SC. Under the real JMM things can only get worse. `ConcurrentHashMap` operations are atomic, and a refresh takes an atomic snapshot of the map. +* **Not modelled:** `descriptionsChanged` (index deltas), `dispose`, Removed/Finish events, races on the `children` list inside `ReferenceSearchViewTreeNode`. + +## Model + +State: S pc, the `listeners` lock, `listening`, `matchingReferences` and its modCount, `rootNodes` (uri ↦ node), `batchAddNodes`, `isUIUpdateScheduled`, scheduled job count, UI pc (with the locals of each UI runnable), pending `syncExec`, viewer input and root items, plus ghost data: ref/node → (uri, epoch), ref→node attachments, the reset epoch, `resetPending`, and `cme`. + +* Every access that is not atomic is its own step. For example, `rootNodes.get` (158), `put` (160-161), batch add (162-164), attach child (137), flag read (173), flag set (174) and schedule (175) are all separate. The UIUpdater's snapshot (207-215), refresh (216), `isEmpty` (217) and flag write/reschedule (218/220) are separate too. +* The `inputChanged` repopulation loop (118-120) walks the live list, with the `ArrayList.Itr` rules: `hasNext` is `cursor != size`, and `next` throws CME if modCount changed. +* Environment nondeterminism: S can pick `accept(ref in any uri)` or `reset()` within the budgets, and the user can switch A→B or B→A within the budget. +* Default bound: 2 URIs, 3 accepts, 1 reset, 2 switches. + +## Properties + +| Id | Statement | Checked where | +|---|---|---| +| P1 | No lost update. At quiescence (S idle, UI idle, no scheduled job, no pending `syncExec`, input A), every reference in `matchingReferences` is attached to a root node that is shown. | quiescent states | +| P1' | Root-level form of P1: at quiescence every node in `rootNodes` is shown. | quiescent states | +| P2 | One root per URI: no two shown root nodes have the same URI. | between UI runnables | +| P3 | No stale node. When input is B, nothing is shown. When input is A and no Reset is still in flight, every shown node belongs to the current epoch. | between UI runnables | +| P4 | No `ConcurrentModificationException` on the UI thread. | all states | +| DL | No deadlock: no state without successors while some actor is mid-flight. | all states | + +## Results + +| Model | States | Edges | P1 | P1' | P2 | P3 | P4 | DL | +|---|---|---|---|---|---|---|---|---| +| **as written** | 635,022 | 1,020,263 | ✗ (20) | ✗ (20) | ✓ | ✗ (12) | ✗ (8) | ✗ (3) | +| patch `lost` | 523,078 | 832,155 | ✗ (28) | ✓ | ✓ | ✗ (10) | ✗ (8) | ✗ (3) | +| patch `lost+order+snap` | 403,838 | 700,869 | ✗ (25) | ✓ | ✓ | ✓ | ✓ | ✗ (3) | +| patch `…+async` | 802,778 | 1,421,456 | ✗ (13) | ✓ | ✓ | ✓ | ✓ | ✓ | +| **fixed** | 9,952 | 16,835 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| fixed, bound 3 URIs / 4 accepts / 2 resets / 3 switches | 1,892,441 | 3,418,927 | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| **fixed + planted bug** | 6,472 | 9,311 | ✗ (2) | ✗ (2) | ✓ | ✓ | ✓ | ✓ | + +✗ (k) = violated, with a shortest counterexample of k steps (BFS order). Every run reached a fixpoint. Full traces are in `report.txt`. + +### Proved and bounded-checked + +* **Proved for all sizes** (any number of URIs, accepts, resets, switches): `Proof.no_lost_root`. In every reachable state of the **fixed** model, if no UIUpdater is scheduled or running, every root node in `rootNodes` is shown (P1', the roots-level form). + * Inductive invariant: `batch ≠ [] → flag`, `flag → jobs > 0 ∨ UI at decide`, and `rootNodes.values ⊆ viewer ∪ batch`. + * Axioms: `propext` and `Quot.sound` only. +* **Kernel-checked bounded results** (`native_decide`, default bound): `Theorems.buggy_verdicts`, `fixed_verdicts` and `planted_caught` pin the verdict vectors in the table above. +* **Bounded only:** P1 (refs level), P2, P3, P4 and DL. + +### Sanity checks + +* The fixed model passes every property at both bounds. +* The planted bug (flag set without scheduling the job) is caught by P1/P1' in 2 steps. +* By construction, the planted bug also breaks clause 2 of the proof's invariant (flag set with no job). This is not separately proved. + +## Findings + +### F1: Lost update on the UIUpdater hand-off. **CONFIRMED** +Shortest trace (20 steps). Condensed from step 10 on, after node0 was delivered and a UIUpdater is scheduled: + +| # | Actor | Step | Code | +|---|---|---|---| +| 10-12 | S | `accept(ref1 in uri1)`, takes lock, `rootNodes.get(uri1)` = null | RSR.accept, FRS:170, FRS:158 | +| 13 | UI | UIUpdater snapshots and clears the batch [node0], `viewer.add` | FRS:207-215 | +| 14 | UI | `viewer.refresh()` | FRS:216 | +| 15 | S | `rootNodes.put(uri1, node1)` | FRS:160-161 | +| 16 | UI | `batchAddNodes.isEmpty()` = **true** | FRS:217 | +| 17-18 | S | `batchAddNodes.add(node1)`; attach ref1 | FRS:162-164, FRS:137 | +| 19 | S | `isUIUpdateScheduled == true`, so no schedule | FRS:173 | +| 20 | UI | `isUIUpdateScheduled = false` | FRS:220 | + +End state: node1 is in `rootNodes` and in the batch, but not in the viewer. The flag is false and no job exists. + +* Nothing brings node1 into the viewer until the next `Added` event. +* If this was the last match of the search, node1 never appears: the Finish event is ignored and nothing else refreshes the viewer. +* Why the Java does this: the "batch empty?" check at 217 and the flag clear at 220 are not atomic with S's batch add (162-164) and flag read (173). + +### F2: Deadlock between Reset's `syncExec` and `inputChanged`. **CONFIRMED** (timing window) + +| # | Actor | Step | Code | +|---|---|---|---| +| 1 | S | `reset()` | RSR.reset, from ReferenceQuery.run at job start | +| 2 | S | `fireEvent(Reset)` holds `A.listeners`, then `Display.syncExec` waits for the UI | FRS:178 | +| 3 | UI | `setInput(B)` → `inputChanged(A,B)` → `A.removeListener` blocks on `A.listeners` | FRS:113 | + +* How to trigger: the user starts another find-references (B) while A's job is starting. `InternalSearchUI.runSearchInBackground(B)` → `SearchViewManager.showNewSearchQuery` → `SearchView.showSearchResult` → `page.setInput(B)` runs on the UI thread. Picking another result from the search history also works. +* Result: S waits for the UI and the UI waits for S. The workbench freezes. +* Re-running A itself is safe, because `setInput(A)` happens before `job.schedule()`. +* The Xtext superclass has no `syncExec` and no such deadlock. + +### F3: Another search's in-flight node shown under result B. **CONFIRMED** +Shortest trace (12 steps): + +| # | Actor | Step | Code | +|---|---|---|---| +| 1-3 | S | `accept(ref0 in uri0)`, lock, `get(uri0)` = null | RSR.accept, FRS:170, FRS:158 | +| 4 | UI | `inputChanged(A,B)`: `rootNodes.clear()` | FRS:111 | +| 5-10 | S | `put(uri0, node0)`, batch add, attach, flag, schedule, release lock | FRS:160-175 | +| 11 | UI | `A.removeListener` (it was waiting for the lock) | FRS:113 | +| 12 | UI | `setInput(B)` refresh | FRS:216 semantics | + +End state: B's view shows A's node0. + +* Cause: the clear (111) comes before the listener removal (113). An A handler that is already in flight writes into the cleared map, which now belongs to B. +* If B is a fresh search, B's own Reset runnable removes the node shortly after, so it is transient. +* If B was picked from history (not re-run), the node stays for good: it is in `rootNodes`, and A's still-scheduled UIUpdater also refreshes it into view. + +### F4: CME on the UI thread in `inputChanged`. **CONFIRMED** +Shortest trace (8 steps): + +| # | Actor | Step | Code | +|---|---|---|---| +| 1-3 | UI | switch to B | | +| 4-5 | UI | switch back: `inputChanged(B,A)`, `A.addListener` | FRS:116 | +| 6 | UI | `getMatchingReferences().iterator()` | FRS:118 | +| 7 | S | `accept`: `matchingReferences.add`, no lock | RSR.accept | +| 8 | UI | `next()` throws `ConcurrentModificationException` out of `viewer.setInput` | FRS:118 | + +* How to trigger: switching the Search view back to a search that is still running. +* A `reset()` (`clear`) during the loop does the same. +* The Xtext superclass has the same live iteration. + +### F5: `resourceNode` check-then-act races the `inputChanged` repopulation. **CONFIRMED** +This was hidden behind F1. It showed up once F1 was patched (28 steps; see `report.txt`, section "patch: lost"). Every step in the trace behaves the same in the unpatched code. + +| # | Actor | Step | Code | +|---|---|---|---| +| — | S | `accept(ref1 in uri0)`: added to the list *before* A.addListener; fired *after* it | RSR.accept | +| 10 | S | `get(uri0)` = null | FRS:158 | +| 13-14 | UI | loop for ref0: `get(uri0)` = null; `put(uri0, node0)` | FRS:119→158, FRS:160-161 | +| 15-17 | S | `put(uri0, node1)` (overwrites node0); batch add; attach ref1→node1 | FRS:160-161, 162-164, 137 | +| 19-20 | UI | batch add node0; attach ref0→node0 | FRS:119 | +| 21-23 | UI | loop for ref1: `get(uri0)` = node1; attach ref1→node1 **again** | FRS:119 | + +End state: + +* Two root nodes were created for uri0. Only node1 is left in `rootNodes` and the viewer. +* ref0 hangs under the orphaned node0 and is never shown (P1 violated). +* ref1 is shown twice (duplicate row). +* Cause: FRS:158-161 is not atomic, and both the UI (inputChanged) and S call it. A reference that is accepted before the iterator is created but fired after `addListener` gets processed twice. +* Trigger: the same history-switch as F4, without the CME. + +### Observations (not violations) +* **Reset does not clear `batchAddNodes` (FRS:182-183).** The next UIUpdater re-adds pre-reset nodes at FRS:213. The full `refresh()` at FRS:216 removes them again inside the same UI runnable, so they are never visible: P3 holds on this path in every variant. The code is correct only because of that refresh. +* **P2 holds everywhere.** `rootNodes` is keyed by URI and every UI runnable ends with a full refresh. F5 still shows that two node objects can be created for one URI. +* **`asyncExec` alone is not a fix for F2.** Swapping `syncExec` for `asyncExec` removes the deadlock, but the Reset runnable then clears nodes added *after* the reset, which loses them (P1, 13 steps). The fixed design handles Reset on S under the provider lock. +* **Not modelled:** if `runInUIThread` throws (for example on a disposed viewer), `isUIUpdateScheduled` stays true for good and this FRS instance never schedules again. + +### Model artefact (found and fixed) +In the `lost+order+snap` variant, P3 first failed because the ghost `resetPending` was cleared at the end of `inputChanged`, even though a Reset event was still in flight. The real code would clear the node when that Reset arrived. The fix to the ghost: a refresh counts as reflecting the reset only when no Reset is in flight (`resetInFlight`). Verdict for that first trace: **MODEL-ARTEFACT**. No other violation above depends on ghost data except through P3, and F3's end state has input = B, which needs no ghost. + +## Fixed design (what `Fixed.lean` checks) +1. `addReference` / `resourceNode` and the flag test-and-set run atomically under one provider lock (dedupe: skip a ref already attached to the current root). UIUpdater's "batch empty ? clear flag : reschedule" runs under the same lock. +2. Reset is handled on S under that lock: clear `rootNodes` and the batch, and make sure an updater is scheduled. No `syncExec`. +3. `inputChanged` calls `removeListener` before clearing. On the way back it calls `addListener`, then snapshots and rebuilds under the provider lock and refreshes. + +## Wall time +About 27 minutes end to end: about 6 minutes reading code and framework sources, about 21 minutes modelling, checking, proving and writing. Machine time: `lake build` about 65 s, `Report.lean` about 2 min 15 s. diff --git a/formal/find-refs/lean/Report.lean b/formal/find-refs/lean/Report.lean new file mode 100644 index 0000000000..3f76d4493d --- /dev/null +++ b/formal/find-refs/lean/Report.lean @@ -0,0 +1,11 @@ +import FindRefs +open FindRefs + +def cfg : Cfg := {} +#eval report "buggy (as written)" (Buggy.next .none cfg) (St.init cfg) +#eval report "patch: lost" (Buggy.next {lost := true} cfg) (St.init cfg) +#eval report "patch: lost+order+snap" (Buggy.next {lost := true, order := true, snap := true} cfg) (St.init cfg) +#eval report "patch: lost+order+snap+async" (Buggy.next {lost := true, order := true, snap := true, async := true} cfg) (St.init cfg) +#eval report "fixed" (Fixed.next false cfg) (St.init cfg) +#eval report "fixed + planted bug" (Fixed.next true cfg) (St.init cfg) +#eval report "fixed, larger bound nU=3 adds=4 resets=2 switches=3" (Fixed.next false {nU := 3, adds := 4, resets := 2, switches := 3}) (St.init {nU := 3, adds := 4, resets := 2, switches := 3}) diff --git a/formal/find-refs/lean/lake-manifest.json b/formal/find-refs/lean/lake-manifest.json new file mode 100644 index 0000000000..172cc6ac5a --- /dev/null +++ b/formal/find-refs/lean/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.2.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "findrefs", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/formal/find-refs/lean/lakefile.toml b/formal/find-refs/lean/lakefile.toml new file mode 100644 index 0000000000..28dca5d382 --- /dev/null +++ b/formal/find-refs/lean/lakefile.toml @@ -0,0 +1,5 @@ +name = "findrefs" +defaultTargets = ["FindRefs"] + +[[lean_lib]] +name = "FindRefs" diff --git a/formal/find-refs/lean/lean-toolchain b/formal/find-refs/lean/lean-toolchain new file mode 100644 index 0000000000..acc704ffe6 --- /dev/null +++ b/formal/find-refs/lean/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.35.0-rc2 diff --git a/formal/find-refs/lean/report.txt b/formal/find-refs/lean/report.txt new file mode 100644 index 0000000000..03859074f4 --- /dev/null +++ b/formal/find-refs/lean/report.txt @@ -0,0 +1,221 @@ +== buggy (as written): 635022 states, 1020263 edges, fixpoint=true + P1 no lost update (refs): VIOLATED, shortest trace (20 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. S RSR.fireEvent(Added ref0) takes listeners lock -> FRS:170 + 3. S FRS:158 rootNodes.get(uri0) = null + 4. S FRS:160-161 rootNodes.put(uri0, node0) + 5. S FRS:162-164 batchAddNodes.add(node0) + 6. S FRS:137 attach ref0 under node0 + 7. S FRS:173 isUIUpdateScheduled == false + 8. S FRS:174 isUIUpdateScheduled = true + 9. S FRS:175 new UIUpdater().schedule(); release listeners lock + 10. S RSR.accept(ref1 in uri1): matchingReferences.add + 11. S RSR.fireEvent(Added ref1) takes listeners lock -> FRS:170 + 12. S FRS:158 rootNodes.get(uri1) = null + 13. UI UIUpdater starts: FRS:207-210 snapshot+clear batch [0]; FRS:212-215 viewer.add each + 14. UI FRS:216 viewer.refresh() (root items := rootNodes.values) + 15. S FRS:160-161 rootNodes.put(uri1, node1) + 16. UI FRS:217 batchAddNodes.isEmpty() = true + 17. S FRS:162-164 batchAddNodes.add(node1) + 18. S FRS:137 attach ref1 under node1 + 19. S FRS:173 isUIUpdateScheduled == true -> skip schedule; release listeners lock + 20. UI FRS:220 isUIUpdateScheduled = false + final: roots=[(some 0), (some 1)] batch=[1] flag=false jobs=0 viewer=[0] matching=[0, 1] attach=[(0, 0), (1, 1)] nodes=[(0, 0), (1, 0)] epoch=0 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P1' no lost update (roots): VIOLATED, shortest trace (20 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. S RSR.fireEvent(Added ref0) takes listeners lock -> FRS:170 + 3. S FRS:158 rootNodes.get(uri0) = null + 4. S FRS:160-161 rootNodes.put(uri0, node0) + 5. S FRS:162-164 batchAddNodes.add(node0) + 6. S FRS:137 attach ref0 under node0 + 7. S FRS:173 isUIUpdateScheduled == false + 8. S FRS:174 isUIUpdateScheduled = true + 9. S FRS:175 new UIUpdater().schedule(); release listeners lock + 10. S RSR.accept(ref1 in uri1): matchingReferences.add + 11. S RSR.fireEvent(Added ref1) takes listeners lock -> FRS:170 + 12. S FRS:158 rootNodes.get(uri1) = null + 13. UI UIUpdater starts: FRS:207-210 snapshot+clear batch [0]; FRS:212-215 viewer.add each + 14. UI FRS:216 viewer.refresh() (root items := rootNodes.values) + 15. S FRS:160-161 rootNodes.put(uri1, node1) + 16. UI FRS:217 batchAddNodes.isEmpty() = true + 17. S FRS:162-164 batchAddNodes.add(node1) + 18. S FRS:137 attach ref1 under node1 + 19. S FRS:173 isUIUpdateScheduled == true -> skip schedule; release listeners lock + 20. UI FRS:220 isUIUpdateScheduled = false + final: roots=[(some 0), (some 1)] batch=[1] flag=false jobs=0 viewer=[0] matching=[0, 1] attach=[(0, 0), (1, 1)] nodes=[(0, 0), (1, 0)] epoch=0 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P2 one root per URI: holds + P3 no stale node: VIOLATED, shortest trace (12 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. S RSR.fireEvent(Added ref0) takes listeners lock -> FRS:170 + 3. S FRS:158 rootNodes.get(uri0) = null + 4. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + 5. S FRS:160-161 rootNodes.put(uri0, node0) + 6. S FRS:162-164 batchAddNodes.add(node0) + 7. S FRS:137 attach ref0 under node0 + 8. S FRS:173 isUIUpdateScheduled == false + 9. S FRS:174 isUIUpdateScheduled = true + 10. S FRS:175 new UIUpdater().schedule(); release listeners lock + 11. UI FRS:113 A.removeListener(this) + 12. UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh + final: roots=[(some 0), none] batch=[0] flag=true jobs=1 viewer=[0] matching=[0] attach=[(0, 0)] nodes=[(0, 0)] epoch=0 input=FindRefs.Inp.B spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P4 no UI-thread CME: VIOLATED, shortest trace (8 steps): + 1. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + 2. UI FRS:113 A.removeListener(this) + 3. UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh + 4. UI user shows result A again: setInput(A) -> inputChanged(B,A) FRS:111 rootNodes.clear + 5. UI FRS:116 A.addListener(this) + 6. UI FRS:118 matchingReferences.iterator() + 7. S RSR.accept(ref0 in uri0): matchingReferences.add + 8. UI FRS:118 iterator.next() throws ConcurrentModificationException + final: roots=[none, none] batch=[] flag=false jobs=0 viewer=[] matching=[0] attach=[] nodes=[] epoch=0 input=FindRefs.Inp.B spc=FindRefs.SPc.fire 0 upc=FindRefs.UPc.idle lockS=false syncReq=false + DL no deadlock: VIOLATED, shortest trace (3 steps): + 1. S RSR.reset(): matchingReferences.clear [search (re)started] + 2. S RSR.fireEvent(Reset) takes listeners lock -> FRS:178 Display.syncExec (blocks) + 3. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + final: roots=[none, none] batch=[] flag=false jobs=0 viewer=[] matching=[] attach=[] nodes=[] epoch=1 input=FindRefs.Inp.A spc=FindRefs.SPc.rwait upc=FindRefs.UPc.wRemove lockS=true syncReq=true +== patch: lost: 523078 states, 832155 edges, fixpoint=true + P1 no lost update (refs): VIOLATED, shortest trace (28 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + 3. UI FRS:113 A.removeListener(this) + 4. S RSR.fireEvent(Added ref0): FRS not a listener, dropped + 5. S RSR.accept(ref1 in uri0): matchingReferences.add + 6. UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh + 7. UI user shows result A again: setInput(A) -> inputChanged(B,A) FRS:111 rootNodes.clear + 8. UI FRS:116 A.addListener(this) + 9. S RSR.fireEvent(Added ref1) takes listeners lock -> FRS:170 + 10. S FRS:158 rootNodes.get(uri0) = null + 11. UI FRS:118 matchingReferences.iterator() + 12. UI FRS:118 next() = ref0 + 13. UI FRS:119->158 rootNodes.get(uri0) = null + 14. UI FRS:119->160-161 rootNodes.put(uri0, node0) + 15. S FRS:160-161 rootNodes.put(uri0, node1) + 16. S FRS:162-164 batchAddNodes.add(node1) + 17. S FRS:137 attach ref1 under node1 + 18. S [patched] synchronized(batch){flag := true; schedule}; release listeners lock + 19. UI FRS:119->162-164 batchAddNodes.add(node0) + 20. UI FRS:119->137 attach ref0 under node0 + 21. UI FRS:118 next() = ref1 + 22. UI FRS:119->158 rootNodes.get(uri0) = node1 + 23. UI FRS:119->137 attach ref1 under node1 + 24. UI FRS:118 iterator.hasNext() = false + 25. UI ContentViewer.setInput(A): input := A; refresh + 26. UI UIUpdater starts: FRS:207-210 snapshot+clear batch [1, 0]; FRS:212-215 viewer.add each + 27. UI FRS:216 viewer.refresh() (root items := rootNodes.values) + 28. UI [patched] synchronized(batch){empty -> flag := false} + final: roots=[(some 1), none] batch=[] flag=false jobs=0 viewer=[1] matching=[0, 1] attach=[(1, 1), (0, 0), (1, 1)] nodes=[(0, 0), (0, 0)] epoch=0 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P1' no lost update (roots): holds + P2 one root per URI: holds + P3 no stale node: VIOLATED, shortest trace (10 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. S RSR.fireEvent(Added ref0) takes listeners lock -> FRS:170 + 3. S FRS:158 rootNodes.get(uri0) = null + 4. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + 5. S FRS:160-161 rootNodes.put(uri0, node0) + 6. S FRS:162-164 batchAddNodes.add(node0) + 7. S FRS:137 attach ref0 under node0 + 8. S [patched] synchronized(batch){flag := true; schedule}; release listeners lock + 9. UI FRS:113 A.removeListener(this) + 10. UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh + final: roots=[(some 0), none] batch=[0] flag=true jobs=1 viewer=[0] matching=[0] attach=[(0, 0)] nodes=[(0, 0)] epoch=0 input=FindRefs.Inp.B spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P4 no UI-thread CME: VIOLATED, shortest trace (8 steps): + 1. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + 2. UI FRS:113 A.removeListener(this) + 3. UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh + 4. UI user shows result A again: setInput(A) -> inputChanged(B,A) FRS:111 rootNodes.clear + 5. UI FRS:116 A.addListener(this) + 6. UI FRS:118 matchingReferences.iterator() + 7. S RSR.accept(ref0 in uri0): matchingReferences.add + 8. UI FRS:118 iterator.next() throws ConcurrentModificationException + final: roots=[none, none] batch=[] flag=false jobs=0 viewer=[] matching=[0] attach=[] nodes=[] epoch=0 input=FindRefs.Inp.B spc=FindRefs.SPc.fire 0 upc=FindRefs.UPc.idle lockS=false syncReq=false + DL no deadlock: VIOLATED, shortest trace (3 steps): + 1. S RSR.reset(): matchingReferences.clear [search (re)started] + 2. S RSR.fireEvent(Reset) takes listeners lock -> FRS:178 Display.syncExec (blocks) + 3. UI user shows other result B: setInput(B) -> inputChanged(A,B) FRS:111 rootNodes.clear + final: roots=[none, none] batch=[] flag=false jobs=0 viewer=[] matching=[] attach=[] nodes=[] epoch=1 input=FindRefs.Inp.A spc=FindRefs.SPc.rwait upc=FindRefs.UPc.wRemove lockS=true syncReq=true +== patch: lost+order+snap: 403838 states, 700869 edges, fixpoint=true + P1 no lost update (refs): VIOLATED, shortest trace (25 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. UI [patched] user shows other result B: inputChanged(A,B) removeListener first + 3. UI [patched] A.removeListener(this); rootNodes.clear + 4. S RSR.fireEvent(Added ref0): FRS not a listener, dropped + 5. UI FRS:116 B.addListener; ContentViewer.setInput(B) refresh + 6. UI user shows result A again: setInput(A) -> inputChanged(B,A) FRS:111 rootNodes.clear + 7. UI FRS:116 A.addListener(this) + 8. UI [patched] snapshot = copy of matchingReferences + 9. S RSR.accept(ref1 in uri0): matchingReferences.add + 10. S RSR.fireEvent(Added ref1) takes listeners lock -> FRS:170 + 11. S FRS:158 rootNodes.get(uri0) = null + 12. UI FRS:118 next() = ref0 + 13. UI FRS:119->158 rootNodes.get(uri0) = null + 14. S FRS:160-161 rootNodes.put(uri0, node0) + 15. S FRS:162-164 batchAddNodes.add(node0) + 16. S FRS:137 attach ref1 under node0 + 17. S [patched] synchronized(batch){flag := true; schedule}; release listeners lock + 18. UI FRS:119->160-161 rootNodes.put(uri0, node1) + 19. UI FRS:119->162-164 batchAddNodes.add(node1) + 20. UI FRS:119->137 attach ref0 under node1 + 21. UI FRS:118 iterator.hasNext() = false + 22. UI ContentViewer.setInput(A): input := A; refresh + 23. UI UIUpdater starts: FRS:207-210 snapshot+clear batch [0, 1]; FRS:212-215 viewer.add each + 24. UI FRS:216 viewer.refresh() (root items := rootNodes.values) + 25. UI [patched] synchronized(batch){empty -> flag := false} + final: roots=[(some 1), none] batch=[] flag=false jobs=0 viewer=[1] matching=[0, 1] attach=[(1, 0), (0, 1)] nodes=[(0, 0), (0, 0)] epoch=0 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P1' no lost update (roots): holds + P2 one root per URI: holds + P3 no stale node: holds + P4 no UI-thread CME: holds + DL no deadlock: VIOLATED, shortest trace (3 steps): + 1. S RSR.reset(): matchingReferences.clear [search (re)started] + 2. S RSR.fireEvent(Reset) takes listeners lock -> FRS:178 Display.syncExec (blocks) + 3. UI [patched] user shows other result B: inputChanged(A,B) removeListener first + final: roots=[none, none] batch=[] flag=false jobs=0 viewer=[] matching=[] attach=[] nodes=[] epoch=1 input=FindRefs.Inp.A spc=FindRefs.SPc.rwait upc=FindRefs.UPc.wRemove lockS=true syncReq=true +== patch: lost+order+snap+async: 802778 states, 1421456 edges, fixpoint=true + P1 no lost update (refs): VIOLATED, shortest trace (13 steps): + 1. S RSR.reset(): matchingReferences.clear [search (re)started] + 2. S [patched] RSR.fireEvent(Reset) -> Display.asyncExec (no wait) + 3. S RSR.accept(ref0 in uri0): matchingReferences.add + 4. S RSR.fireEvent(Added ref0) takes listeners lock -> FRS:170 + 5. S FRS:158 rootNodes.get(uri0) = null + 6. S FRS:160-161 rootNodes.put(uri0, node0) + 7. S FRS:162-164 batchAddNodes.add(node0) + 8. S FRS:137 attach ref0 under node0 + 9. S [patched] synchronized(batch){flag := true; schedule}; release listeners lock + 10. UI UIUpdater starts: FRS:207-210 snapshot+clear batch [0]; FRS:212-215 viewer.add each + 11. UI FRS:216 viewer.refresh() (root items := rootNodes.values) + 12. UI [patched] synchronized(batch){empty -> flag := false} + 13. UI Reset runnable FRS:181-186: viewer.remove(rootNodes.values), rootNodes.clear, refresh + final: roots=[none, none] batch=[] flag=false jobs=0 viewer=[] matching=[0] attach=[(0, 0)] nodes=[(0, 1)] epoch=1 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P1' no lost update (roots): holds + P2 one root per URI: holds + P3 no stale node: holds + P4 no UI-thread CME: holds + DL no deadlock: holds +== fixed: 9952 states, 16835 edges, fixpoint=true + P1 no lost update (refs): holds + P1' no lost update (roots): holds + P2 one root per URI: holds + P3 no stale node: holds + P4 no UI-thread CME: holds + DL no deadlock: holds +== fixed + planted bug: 6472 states, 9311 edges, fixpoint=true + P1 no lost update (refs): VIOLATED, shortest trace (2 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. S fireEvent(Added ref0) -> atomic addReference + final: roots=[(some 0), none] batch=[0] flag=true jobs=0 viewer=[] matching=[0] attach=[(0, 0)] nodes=[(0, 0)] epoch=0 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P1' no lost update (roots): VIOLATED, shortest trace (2 steps): + 1. S RSR.accept(ref0 in uri0): matchingReferences.add + 2. S fireEvent(Added ref0) -> atomic addReference + final: roots=[(some 0), none] batch=[0] flag=true jobs=0 viewer=[] matching=[0] attach=[(0, 0)] nodes=[(0, 0)] epoch=0 input=FindRefs.Inp.A spc=FindRefs.SPc.idle upc=FindRefs.UPc.idle lockS=false syncReq=false + P2 one root per URI: holds + P3 no stale node: holds + P4 no UI-thread CME: holds + DL no deadlock: holds +== fixed, larger bound nU=3 adds=4 resets=2 switches=3: 1892441 states, 3418927 edges, fixpoint=true + P1 no lost update (refs): holds + P1' no lost update (roots): holds + P2 one root per URI: holds + P3 no stale node: holds + P4 no UI-thread CME: holds + DL no deadlock: holds +lake env lean Report.lean 130.83s user 0.71s system 99% cpu 2:12.56 total diff --git a/formal/find-refs/lean/theorems.txt b/formal/find-refs/lean/theorems.txt new file mode 100644 index 0000000000..1200745870 --- /dev/null +++ b/formal/find-refs/lean/theorems.txt @@ -0,0 +1,16 @@ +# ; checked by formal/check.sh via #print axioms +FindRefs.Proof.addRef_inv kernel +FindRefs.Proof.attachRef_cover kernel +FindRefs.Proof.attachRef_frame kernel +FindRefs.Proof.ensureScheduled_inv kernel +FindRefs.Proof.init_inv kernel +FindRefs.Proof.no_lost_root kernel +FindRefs.Proof.reach_inv kernel +FindRefs.Proof.rebuild_frame kernel +FindRefs.Proof.rootsVals_clear kernel +FindRefs.Proof.rootsVals_set kernel +FindRefs.Proof.searchStarts_frame kernel +FindRefs.Proof.step_inv kernel +FindRefs.Theorems.buggy_verdicts native_decide +FindRefs.Theorems.fixed_verdicts native_decide +FindRefs.Theorems.planted_caught native_decide diff --git a/formal/find-refs/tla/FindRefs.cfg b/formal/find-refs/tla/FindRefs.cfg new file mode 100644 index 0000000000..7ffe551ab4 --- /dev/null +++ b/formal/find-refs/tla/FindRefs.cfg @@ -0,0 +1,15 @@ +\* Original code. Checks the three requested properties plus the extras; TLC stops at the first +\* violation (use run.sh / matrix.sh for one-property-per-run shortest traces). +SPECIFICATION Spec +CONSTANTS + Refs <- DefaultRefs + Fixes <- NoFixes + Planted = FALSE + MaxRuns = 2 + MaxSwitch = 2 +INVARIANTS + TypeOK + NoLostRoot NoLostRef + OneRootPerUri OneRootCreated NoDupRef + NoStale NoForeign + NoCME NoDeadlock diff --git a/formal/find-refs/tla/FindRefs.tla b/formal/find-refs/tla/FindRefs.tla new file mode 100644 index 0000000000..1401255ac6 --- /dev/null +++ b/formal/find-refs/tla/FindRefs.tla @@ -0,0 +1,483 @@ +------------------------------ MODULE FindRefs ------------------------------ +(***************************************************************************) +(* Model of FastReferenceSearchResultContentProvider (FRSRCP) together *) +(* with the Xtext ReferenceSearchResult R it listens to, the search job *) +(* thread, the SWT UI thread (UIJob + syncExec/asyncExec runnables + user *) +(* view switches) and the JFace TreeViewer's root items. *) +(* *) +(* Line refs: FRSRCP = FastReferenceSearchResultContentProvider.java, *) +(* RSR = xtext ReferenceSearchResult.java, *) +(* RSVP = xtext ReferenceSearchViewPage.java. *) +(* *) +(* Fixes = {} is the code as written; each element enables one minimal *) +(* fix (see NOTES.md). Planted = TRUE plants an obvious bug on top. *) +(* *) +(***************************************************************************) +EXTENDS Naturals, Sequences, FiniteSets, TLC + +CONSTANTS Refs, \* resource URI of each reference the search delivers, in order + MaxRuns, \* number of search runs (>1 = "Search Again", i.e. Reset on a live provider) + MaxSwitch, \* number of user view switches R->B / B->R (B = another, finished search) + Fixes, \* subset of AllFixes + Planted \* planted bug: the Added handler never schedules the UIUpdater + +AllFixes == {"flag", "atomic", "dedupe", "snapshot", "reset", "plock", "detach"} +ASSUME Fixes \subseteq AllFixes /\ Planted \in BOOLEAN + +Fix(x) == x \in Fixes + +\* Defaults for FindRefs.cfg / FindRefsFixed.cfg (cfg files cannot hold sequences) +DefaultRefs == <<"u1", "u2">> +NoFixes == {} +MinFixes == {"flag", "dedupe", "snapshot", "reset", "plock", "detach"} +URIs == {Refs[k] : k \in DOMAIN Refs} +Nil == 0 +RefUri(ref) == Refs[ref[2]] \* a reference is <> +Range(f) == {f[x] : x \in DOMAIN f} + +VARIABLES + \* --- ReferenceSearchResult R (Xtext) --- + matching, \* R.matchingReferences (ArrayList, unsynchronized) + matchMod, \* its modCount + provReg, \* provider \in R.listeners + lockL, \* monitor of R.listeners: "none" | "S" | "U" + lockP, \* (fix "plock") provider-private monitor: "none" | "S" | "U" + \* --- provider state --- + rootNodes, \* URI -> node id (ConcurrentMap), Nil = absent + batch, \* batchAddNodes + flag, \* isUIUpdateScheduled + nodes, \* heap of ReferenceSearchViewTreeNode: [uri, ep, kids, gen] + clears, \* number of rootNodes.clear() so far (node.gen = clears at creation) + \* --- Display / Jobs --- + jobs, \* number of pending UIUpdater executions + syncPending, \* Reset runnable posted by syncExec, not yet run + asyncQ, \* (fixed) posted asyncExec refresh runnables (epochs) + \* --- viewer --- + shown, \* node ids that are root items of the TreeViewer + view, \* result the page shows: "R" | "B" + provEpoch, \* epoch of the last Reset / inputChanged the viewer has processed + \* --- search thread --- + pcS, iS, epoch, runs, sNode, + \* --- UI thread --- + pcU, uNonEmpty, uCur, uExp, uSnap, uRef, uNode, cme, switches + +vars == <> + +EmptyMap == [u \in URIs |-> Nil] +RootSet(rn) == Range(rn) \ {Nil} +NewNode(u) == [uri |-> u, ep |-> epoch, kids |-> <<>>, gen |-> clears] +HasKid(n, ref) == \E j \in DOMAIN nodes[n].kids : nodes[n].kids[j] = ref +AddKid(ns, n, ref) == \* new DynamicReferenceSearchViewTreeNode(parent, ...) -> parent.addChild + IF Fix("dedupe") /\ \E j \in DOMAIN ns[n].kids : ns[n].kids[j] = ref THEN ns \* fix: dedupe + ELSE [ns EXCEPT ![n].kids = Append(@, ref)] + +Init == + /\ matching = <<>> /\ matchMod = 0 + /\ provReg = TRUE \* page showed R (setInput) before the job was scheduled + /\ lockL = "none" /\ lockP = "none" + /\ rootNodes = EmptyMap /\ batch = <<>> /\ flag = FALSE /\ nodes = <<>> /\ clears = 0 + /\ jobs = 0 /\ syncPending = FALSE /\ asyncQ = <<>> + /\ shown = {} /\ view = "R" /\ provEpoch = 0 + /\ pcS = "sched" /\ iS = 0 /\ epoch = 0 /\ runs = 1 /\ sNode = Nil + /\ pcU = "idle" /\ uNonEmpty = FALSE /\ uCur = 0 /\ uExp = 0 /\ uSnap = <<>> + /\ uRef = <<0, 0>> /\ uNode = Nil /\ cme = FALSE /\ switches = 0 + +UVars == <> +UVarsNoC == <> +UVarsNoP == <> +SVars == <> + +(***************************************************************************) +(* Search job thread (InternalSearchJob -> ReferenceQuery.run). *) +(***************************************************************************) +SJobStart == \* job picked up by a worker + /\ pcS = "sched" /\ pcS' = "reset" + /\ UNCHANGED <> + +SReset == \* RSR:104 matchingReferences.clear() + /\ pcS = "reset" + /\ matching' = <<>> /\ matchMod' = matchMod + 1 /\ epoch' = epoch + 1 + /\ pcS' = "rlock" + /\ UNCHANGED <> + +SResetLock == \* RSR:54 synchronized(listeners) -> FRSRCP:177 Reset handler + /\ pcS = "rlock" /\ lockL = "none" /\ lockL' = "S" + /\ IF provReg /\ ~Fix("reset") + THEN /\ syncPending' = TRUE /\ pcS' = "rwait" \* FRSRCP:178 syncExec (blocks) + /\ UNCHANGED <> + ELSE IF provReg /\ Fix("reset") + THEN \* fix: clear model on the search thread, asyncExec only the viewer refresh + /\ pcS' = "rclear" /\ UNCHANGED <> + ELSE /\ pcS' = "runlock" /\ UNCHANGED <> + /\ UNCHANGED <> + +SRClear == \* fix "reset": clear model on the search thread (under the provider lock if "plock"), + \* and asyncExec only the viewer refresh + /\ pcS = "rclear" /\ (Fix("plock") => lockP = "none") + /\ rootNodes' = EmptyMap /\ clears' = clears + 1 /\ batch' = <<>> /\ asyncQ' = Append(asyncQ, epoch) /\ pcS' = "runlock" + /\ UNCHANGED <> + +SResetWait == \* syncExec returns once the UI thread ran the runnable + /\ pcS = "rwait" /\ ~syncPending /\ pcS' = "runlock" + /\ UNCHANGED <> + +SResetUnlock == + /\ pcS = "runlock" /\ lockL' = "none" + /\ iS' = 1 /\ pcS' = IF Len(Refs) >= 1 THEN "acc" ELSE "fin" + /\ UNCHANGED <> + +SAccept == \* RSR:85 matchingReferences.add (outside the listeners lock) + /\ pcS = "acc" + /\ matching' = Append(matching, <>) /\ matchMod' = matchMod + 1 + /\ pcS' = "lock" + /\ UNCHANGED <> + +SLock == \* RSR:54-56 fireEvent(Added): lock, deliver to provider if registered + /\ pcS = "lock" /\ lockL = "none" /\ lockL' = "S" + /\ pcS' = IF provReg THEN (IF Fix("plock") THEN "plk" ELSE "get") ELSE "unlock" + /\ UNCHANGED <> + +SPLock == \* fix "plock": synchronized (providerLock) around the Added handler body + /\ pcS = "plk" /\ lockP = "none" /\ lockP' = "S" /\ pcS' = "get" + /\ UNCHANGED <> + +SGet == \* FRSRCP:158-160 rootNodes.get; if null new node (fix "atomic": computeIfAbsent; subsumed by "plock") + /\ pcS = "get" + /\ LET u == Refs[iS] IN + IF rootNodes[u] # Nil + THEN /\ sNode' = rootNodes[u] /\ pcS' = "child" + /\ UNCHANGED <> + ELSE /\ nodes' = Append(nodes, NewNode(u)) /\ sNode' = Len(nodes) + 1 + /\ IF Fix("atomic") + THEN /\ rootNodes' = [rootNodes EXCEPT ![u] = Len(nodes) + 1] + /\ batch' = Append(batch, Len(nodes) + 1) /\ pcS' = "child" + ELSE /\ pcS' = "put" /\ UNCHANGED <> + /\ UNCHANGED <> + +SPut == \* FRSRCP:161 rootNodes.put + /\ pcS = "put" /\ rootNodes' = [rootNodes EXCEPT ![Refs[iS]] = sNode] /\ pcS' = "batch" + /\ UNCHANGED <> + +SBatch == \* FRSRCP:162-164 batchAddNodes.add (under its own lock -> atomic) + /\ pcS = "batch" /\ batch' = Append(batch, sNode) /\ pcS' = "child" + /\ UNCHANGED <> + +SChild == \* FRSRCP:137 new DynamicReferenceSearchViewTreeNode(resourceNode, ...) -> addChild + /\ pcS = "child" /\ nodes' = AddKid(nodes, sNode, <>) /\ pcS' = "flag" + /\ UNCHANGED <> + +SFlag == \* FRSRCP:173 if (!isUIUpdateScheduled) + /\ pcS = "flag" + /\ pcS' = IF flag \/ Planted THEN "unlock" ELSE "setflag" + /\ UNCHANGED <> + +SSetFlag == \* FRSRCP:174 + /\ pcS = "setflag" /\ flag' = TRUE /\ pcS' = "sched2" + /\ UNCHANGED <> + +SSchedule == \* FRSRCP:175 new UIUpdater().schedule() + /\ pcS = "sched2" /\ jobs' = jobs + 1 /\ pcS' = "unlock" + /\ UNCHANGED <> + +SUnlock == \* RSR:57 end of fireEvent + /\ pcS = "unlock" /\ lockL' = "none" /\ lockP' = IF lockP = "S" THEN "none" ELSE lockP + /\ IF iS < Len(Refs) THEN iS' = iS + 1 /\ pcS' = "acc" ELSE pcS' = "fin" /\ UNCHANGED iS + /\ UNCHANGED <> + +SFinish == \* RSR:109 fireEvent(Finish): provider ignores it; lock taken and released + /\ pcS = "fin" /\ lockL = "none" /\ pcS' = "done" + /\ UNCHANGED <> + +SearchStep == SJobStart \/ SReset \/ SResetLock \/ SResetWait \/ SResetUnlock \/ SAccept + \/ SLock \/ SGet \/ SPut \/ SBatch \/ SChild \/ SFlag \/ SSetFlag \/ SSchedule \/ SUnlock \/ SFinish \/ SRClear \/ SPLock + +(***************************************************************************) +(* UI thread. Only one runnable / event handler runs at a time; picks any *) +(* pending one when idle. *) +(***************************************************************************) +Unchanged_S_R == UNCHANGED <> + +UIRunSyncReset == \* FRSRCP:180-186 on the UI thread + /\ pcU = "idle" /\ syncPending + /\ shown' = {} \* remove(input, rootNodes.values); rootNodes.clear; refresh -> getElements = {} + /\ rootNodes' = EmptyMap /\ clears' = clears + 1 /\ syncPending' = FALSE /\ provEpoch' = epoch + /\ UNCHANGED <> + /\ UNCHANGED <> + +UIRunAsyncReset == \* (fixed only) asyncExec'd viewer.refresh after Reset + /\ pcU = "idle" /\ asyncQ # <<>> + /\ shown' = RootSet(rootNodes) /\ provEpoch' = Head(asyncQ) /\ asyncQ' = Tail(asyncQ) + /\ UNCHANGED <> + /\ Unchanged_S_R + +UIJobStart == \* UIJob -> Display.asyncExec(runInUIThread) + /\ pcU = "idle" /\ jobs > 0 /\ jobs' = jobs - 1 + /\ pcU' = IF Fix("flag") THEN "u_clr" ELSE "u_drain" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UClear == \* (fixed) isUIUpdateScheduled = false first, as upstream Xtext does + /\ pcU = "u_clr" /\ flag' = FALSE /\ pcU' = "u_drain" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UDrain == \* FRSRCP:207-215 copy+clear batch under lock, viewer.add each + /\ pcU = "u_drain" + /\ shown' = shown \cup Range(batch) /\ batch' = <<>> /\ pcU' = "u_refresh" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URefresh == \* FRSRCP:216 viewer.refresh() -> getElements() = rootNodes.values() + /\ pcU = "u_refresh" + /\ shown' = RootSet(rootNodes) /\ pcU' = IF Fix("flag") THEN "idle" ELSE "u_check" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UCheck == \* FRSRCP:217 !batchAddNodes.isEmpty() (unsynchronized read) + /\ pcU = "u_check" /\ uNonEmpty' = (batch # <<>>) /\ pcU' = "u_end" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UEnd == \* FRSRCP:218 schedule(250) / 220 isUIUpdateScheduled = false + /\ pcU = "u_end" + /\ IF uNonEmpty THEN jobs' = jobs + 1 /\ UNCHANGED flag + ELSE flag' = FALSE /\ UNCHANGED jobs + /\ pcU' = "idle" + /\ UNCHANGED <> + /\ Unchanged_S_R + +\* ---- user: "Search Again" on R (SearchAgainAction -> runQueryInBackground; addQuery is a no-op) ---- +UserRerun == + /\ pcU = "idle" /\ view = "R" /\ pcS = "done" /\ runs < MaxRuns + /\ pcS' = "sched" /\ runs' = runs + 1 + /\ UNCHANGED <> + +\* ---- user: switch the view to another (finished) search B, e.g. from history ---- +\* SearchView.internalShowSearchPage: page.setInput(null) ; page.setInput(B) +UserToB == + /\ pcU = "idle" /\ view = "R" /\ switches < MaxSwitch + /\ pcU' = "b_disp" /\ switches' = switches + 1 + /\ UNCHANGED <> + /\ Unchanged_S_R + +UBDisp == \* event dispatch / getUIState before touching R + /\ pcU = "b_disp" /\ pcU' = "b_lbl" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UBLabel == \* RSVP:139 R.removeListener(labelUpdater): needs R.listeners monitor + /\ pcU = "b_lbl" /\ lockL = "none" + /\ pcU' = IF Fix("detach") THEN "b_rm" ELSE "b_clear" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UBClear == \* FRSRCP:111 rootNodes.clear() + /\ pcU = "b_clear" /\ (Fix("plock") => lockP = "none") /\ rootNodes' = EmptyMap /\ clears' = clears + 1 + /\ pcU' = IF Fix("detach") THEN "b_ref" ELSE "b_rm" + /\ UNCHANGED <> + /\ UNCHANGED <> + +UBRemove == \* FRSRCP:113 R.removeListener(this); fix: done before the clear + /\ pcU = "b_rm" /\ lockL = "none" /\ provReg' = FALSE + /\ pcU' = IF Fix("detach") THEN "b_clear" ELSE "b_ref" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UBRefresh == \* B has no provider-side refs; viewer.setInput(B) ends with refresh() + /\ pcU = "b_ref" /\ shown' = RootSet(rootNodes) /\ view' = "B" /\ pcU' = "idle" + /\ UNCHANGED <> + /\ Unchanged_S_R + +\* ---- user: switch back to R (possibly still running) ---- +UserToR == + /\ pcU = "idle" /\ view = "B" /\ switches < MaxSwitch + /\ pcU' = "r_disp" /\ switches' = switches + 1 + /\ UNCHANGED <> + /\ Unchanged_S_R + +URDisp == + /\ pcU = "r_disp" /\ pcU' = "r_lbl" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URLabel == \* RSVP:144 R.addListener(labelUpdater) + /\ pcU = "r_lbl" /\ lockL = "none" /\ pcU' = IF Fix("plock") THEN "r_add" ELSE "r_clear" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URClear == \* FRSRCP:111 (fix "plock": done after addListener, under the provider lock, + \* together with the snapshot of the matches) + /\ pcU = "r_clear" /\ rootNodes' = EmptyMap /\ clears' = clears + 1 /\ provEpoch' = epoch + /\ pcU' = IF Fix("plock") THEN "r_iter" ELSE "r_add" + /\ uSnap' = IF Fix("plock") /\ Fix("snapshot") THEN matching ELSE uSnap + /\ UNCHANGED <> + /\ UNCHANGED <> + +URAdd == \* FRSRCP:116 R.addListener(this) (fix: snapshot matching under the same lock) + /\ pcU = "r_add" /\ lockL = "none" /\ provReg' = TRUE + /\ uSnap' = IF Fix("snapshot") /\ ~Fix("plock") THEN matching ELSE <<>> + /\ pcU' = IF Fix("plock") THEN "r_plk" ELSE "r_iter" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URPLock == \* fix "plock": synchronized (providerLock) { clear; snapshot; addReference* } + /\ pcU = "r_plk" /\ lockP = "none" /\ lockP' = "U" /\ pcU' = "r_clear" + /\ UNCHANGED <> + /\ UNCHANGED <> + +URIter == \* FRSRCP:118 getMatchingReferences().iterator() + /\ pcU = "r_iter" /\ uCur' = 0 /\ uExp' = matchMod /\ pcU' = "r_next" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URNext == \* ArrayList.Itr.hasNext (cursor != size) / next (modCount check) + /\ pcU = "r_next" + /\ LET src == IF Fix("snapshot") THEN uSnap ELSE matching IN + IF uCur = Len(src) + THEN pcU' = "r_ref" /\ UNCHANGED <> + ELSE IF ~Fix("snapshot") /\ matchMod # uExp + THEN cme' = TRUE /\ pcU' = "idle" /\ UNCHANGED <> \* CME escapes inputChanged + ELSE uRef' = src[uCur + 1] /\ uCur' = uCur + 1 /\ pcU' = "r_get" /\ UNCHANGED cme + /\ lockP' = IF pcU' # "r_get" /\ lockP = "U" THEN "none" ELSE lockP + /\ UNCHANGED <> + /\ UNCHANGED <> + +URGet == \* addReference -> resourceNode: FRSRCP:158-160 (fix "atomic") + /\ pcU = "r_get" + /\ LET u == RefUri(uRef) IN + IF rootNodes[u] # Nil + THEN uNode' = rootNodes[u] /\ pcU' = "r_child" /\ UNCHANGED <> + ELSE /\ nodes' = Append(nodes, [uri |-> u, ep |-> uRef[1], kids |-> <<>>, gen |-> clears]) + /\ uNode' = Len(nodes) + 1 + /\ IF Fix("atomic") + THEN /\ rootNodes' = [rootNodes EXCEPT ![u] = Len(nodes) + 1] + /\ batch' = Append(batch, Len(nodes) + 1) /\ pcU' = "r_child" + ELSE pcU' = "r_put" /\ UNCHANGED <> + /\ UNCHANGED <> + /\ Unchanged_S_R + +URPut == \* FRSRCP:161 + /\ pcU = "r_put" /\ rootNodes' = [rootNodes EXCEPT ![RefUri(uRef)] = uNode] /\ pcU' = "r_batch" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URBatch == \* FRSRCP:162-164 + /\ pcU = "r_batch" /\ batch' = Append(batch, uNode) /\ pcU' = "r_child" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URChild == \* FRSRCP:137 + /\ pcU = "r_child" /\ nodes' = AddKid(nodes, uNode, uRef) /\ pcU' = "r_next" + /\ UNCHANGED <> + /\ Unchanged_S_R + +URRefresh == \* viewer.setInput(R) -> refresh() + /\ pcU = "r_ref" /\ shown' = RootSet(rootNodes) /\ view' = "R" /\ pcU' = "idle" + /\ UNCHANGED <> + /\ Unchanged_S_R + +UIStep == UIRunSyncReset \/ UIRunAsyncReset \/ UIJobStart \/ UClear \/ UDrain \/ URefresh + \/ UCheck \/ UEnd \/ UserRerun + \/ UserToB \/ UBDisp \/ UBLabel \/ UBClear \/ UBRemove \/ UBRefresh + \/ UserToR \/ URDisp \/ URLabel \/ URClear \/ URAdd \/ URPLock \/ URIter \/ URNext + \/ URGet \/ URPut \/ URBatch \/ URChild \/ URRefresh + +Quiescent == pcS = "done" /\ pcU = "idle" /\ jobs = 0 /\ ~syncPending /\ asyncQ = <<>> + +Terminated == Quiescent /\ UNCHANGED vars \* legitimate end; anything else stuck is a deadlock + +Next == SearchStep \/ UIStep \/ Terminated + +Spec == Init /\ [][Next]_vars + +(***************************************************************************) +(* Properties *) +(***************************************************************************) +TypeOK == + /\ lockL \in {"none", "S", "U"} /\ lockP \in {"none", "S", "U"} /\ view \in {"R", "B"} /\ flag \in BOOLEAN + /\ \A u \in URIs : rootNodes[u] \in 0..Len(nodes) + /\ shown \subseteq 1..Len(nodes) + +Settled == Quiescent /\ view = "R" /\ ~cme + +RefShown(ref) == \E n \in shown : HasKid(n, ref) + +\* P1 no lost update: once quiet, every accepted reference (and its resource root) is visible +NoLostRoot == Settled => \A k \in DOMAIN matching : \E n \in shown : nodes[n].uri = RefUri(matching[k]) +NoLostRef == Settled => \A k \in DOMAIN matching : RefShown(matching[k]) + +\* P2 one root per URI (viewer, whenever the UI thread is between events) +OneRootPerUri == pcU = "idle" => \A u \in URIs : Cardinality({n \in shown : nodes[n].uri = u}) <= 1 +\* P2 at provider level: since the last rootNodes.clear() at most one root node was created per URI +OneRootCreated == \A u \in URIs : + Cardinality({n \in 1..Len(nodes) : nodes[n].uri = u /\ nodes[n].gen = clears}) <= 1 +\* P2' each reference appears once +NoDupRef == Settled => + \A n1, n2 \in shown : \A j1 \in DOMAIN nodes[n1].kids : \A j2 \in DOMAIN nodes[n2].kids : + (nodes[n1].kids[j1] = nodes[n2].kids[j2]) => (n1 = n2 /\ j1 = j2) + +\* P3 after a Reset (or re-input) no node from before it is shown, and R's nodes never leak into B +NoStale == pcU = "idle" => \A n \in shown : nodes[n].ep >= provEpoch +NoForeign == (pcU = "idle" /\ view = "B") => shown = {} + +\* Extra: no ConcurrentModificationException in inputChanged, no UI/search deadlock +NoCME == ~cme +NoDeadlock == ~(pcS = "rwait" /\ syncPending /\ lockL = "S" /\ pcU \in {"b_lbl", "b_rm", "r_lbl", "r_add"}) + +(***************************************************************************) +(* Witnesses (expected to be VIOLATED: they prove good paths are reachable) *) +(***************************************************************************) +W_AllShown == ~(Settled /\ Len(matching) = Len(Refs) /\ \A k \in DOMAIN matching : RefShown(matching[k])) +W_RerunShown == ~(Settled /\ runs = 2 /\ Len(matching) = Len(Refs) /\ \A k \in DOMAIN matching : RefShown(matching[k])) +W_SwitchBack == ~(Settled /\ switches = 2 /\ Len(matching) = Len(Refs) /\ \A k \in DOMAIN matching : RefShown(matching[k])) +W_Rescheduled == ~(pcU = "u_end" /\ uNonEmpty) +W_ResetRan == ~(provEpoch = 2 /\ pcU = "idle") +============================================================================= diff --git a/formal/find-refs/tla/FindRefsFixed.cfg b/formal/find-refs/tla/FindRefsFixed.cfg new file mode 100644 index 0000000000..755ee0cc68 --- /dev/null +++ b/formal/find-refs/tla/FindRefsFixed.cfg @@ -0,0 +1,14 @@ +\* Fixed model (all minimal fixes). Must pass, including TLC deadlock checking. +SPECIFICATION Spec +CONSTANTS + Refs <- DefaultRefs + Fixes <- MinFixes + Planted = FALSE + MaxRuns = 2 + MaxSwitch = 2 +INVARIANTS + TypeOK + NoLostRoot NoLostRef + OneRootPerUri OneRootCreated NoDupRef + NoStale NoForeign + NoCME NoDeadlock diff --git a/formal/find-refs/tla/NOTES.md b/formal/find-refs/tla/NOTES.md new file mode 100644 index 0000000000..4a38d359b9 --- /dev/null +++ b/formal/find-refs/tla/NOTES.md @@ -0,0 +1,193 @@ +# FindRefs TLA+ model: FastReferenceSearchResultContentProvider + +This directory holds a blind model of `com.avaloq.tools.ddk.xtext.ui/.../findrefs/FastReferenceSearchResultContentProvider.java` (FRSRCP). It also models the Xtext classes the provider works with: `ReferenceSearchResult` (RSR), `ReferenceQuery` and `ReferenceSearchViewPage` (RSVP), and the Eclipse Search classes `InternalSearchUI`, `SearchView` and `SearchViewManager`. The Xtext sources come from eclipse/xtext at e8855b27fd. The Eclipse Search sources come from the `org.eclipse.search.source_3.19.0` p2 bundle (extract locally; they are not committed). + +## Files +- `FindRefs.tla`: the spec, 413 non-blank lines. `Fixes` (a set of fix names) and `Planted` switch between the original code, the fixed code and the planted bug. +- `FindRefs.cfg` checks the original code. `FindRefsFixed.cfg` checks the model with all minimal fixes. +- `run.sh