From de92c8116d8c8e9b0263de527d7fdb3e900d3713 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Sun, 6 Sep 2026 22:17:12 +0300 Subject: [PATCH 1/6] feat(semantic_snapshot): narrow a snapshot by identifier prefix, subtree and fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A snapshot of a busy screen repeated the viewport rectangle on every one of its nodes and offered no way to ask for less than the whole tree. The viewport is now stated once, in the envelope (a single-ref visibility answer keeps its own copy, having no envelope), and three optional arguments narrow what comes back: - identifierPrefix keeps nodes whose Semantics identifier starts with it; - subtreeOf keeps one node and its descendants, named by a ref from the latest snapshot or by an identifier, the ref tried first — an unknown root is refused as subtree_root_not_found before any snapshot is spent, so the caller's refs and snapshot_id stay valid; - fields projects each node to the named keys, ref always included; a name outside the node keys is refused as unknown_field, by the server catalog before the round trip and by the app after it. The tree is always walked whole, so refs on a filtered snapshot are the refs of the full one and every interaction tool resolves them. A filtered reply adds totalNodeCount and echoes the filter; children lists kept refs only; interactionSurface still describes the app rather than the slice. A node that carries only an identifier now counts as meaningful: a container named "rail" or "panel" exists to be addressed, and subtreeOf by ref needs it listed. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Fao9J2ibPNtCFhmMRjXv96 --- .../lib/src/shared_core/command_executor.dart | 12 +- .../commands/commands_catalog.dart | 25 +- mcp_server_dart/lib/src/skill_assets.g.dart | 15 +- .../test/command_catalog_test.dart | 27 ++ .../services/semantic_snapshot_service.dart | 208 ++++++++++++++- .../lib/src/toolkits/interaction_toolkit.dart | 25 +- ...nteraction_toolkit_schema_parity_test.dart | 6 + .../test/semantic_snapshot_surface_test.dart | 252 ++++++++++++++++++ .../core/lib/src/commands/core_commands.dart | 10 +- .../src/tools/interaction_input_schemas.dart | 54 +++- .../flutter-mcp-toolkit-control/SKILL.md | 2 +- .../flutter-mcp-toolkit-inspect/SKILL.md | 13 +- 12 files changed, 623 insertions(+), 26 deletions(-) diff --git a/mcp_server_dart/lib/src/shared_core/command_executor.dart b/mcp_server_dart/lib/src/shared_core/command_executor.dart index 1bf8d34f..a0e09510 100644 --- a/mcp_server_dart/lib/src/shared_core/command_executor.dart +++ b/mcp_server_dart/lib/src/shared_core/command_executor.dart @@ -331,7 +331,7 @@ final class DefaultCoreCommandExecutor implements CoreCommandExecutor { GetViewDetailsCommand() => _getViewDetails(), InspectWidgetAtPointCommand() => _inspectWidgetAtPoint(command), CaptureUiSnapshotCommand() => _captureUiSnapshot(command), - SemanticSnapshotCommand() => _semanticSnapshot(), + SemanticSnapshotCommand() => _semanticSnapshot(command), TapWidgetCommand() => _tapWidget(command), EnterTextCommand() => _enterText(command), RevealSearchCommand() => _revealSearch(command), @@ -930,13 +930,21 @@ final class DefaultCoreCommandExecutor implements CoreCommandExecutor { } } - Future _semanticSnapshot() async { + Future _semanticSnapshot( + final SemanticSnapshotCommand command, + ) async { final ensureFailure = await _ensureVmConnected(); if (ensureFailure != null) return ensureFailure; try { final result = await connectionContext.callFlutterExtension( mcpToolkitExtKeys.semanticSnapshot, + args: { + if (command.identifierPrefix != null) + 'identifierPrefix': command.identifierPrefix, + if (command.subtreeOf != null) 'subtreeOf': command.subtreeOf, + if (command.fields != null) 'fields': command.fields, + }, ); return CoreResult.success(data: _map(result.json)); } on Exception catch (e) { diff --git a/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart b/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart index 380ddbe1..637693e2 100644 --- a/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart +++ b/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart @@ -674,7 +674,30 @@ final class CommandCatalog { requiresVm: true, supportsWatch: true, mcpExposed: true, - build: (final args) => const SemanticSnapshotCommand(), + build: (final args) { + // Schema validation has already rejected anything but an array; + // the enum inside it is not enforced there, so name a bad field + // here rather than after a VM round trip. + final rawFields = _findArg(args, 'fields'); + final fields = rawFields is List + ? rawFields.map((final f) => f.toString()).toList() + : null; + final unknown = fields + ?.where((final f) => !semanticSnapshotNodeFields.contains(f)) + .toList(); + if (unknown != null && unknown.isNotEmpty) { + throw ArgumentError( + 'Invalid value for "fields": ${unknown.join(', ')} ' + '(accepted: ${semanticSnapshotNodeFields.join(', ')}; ' + r'schema path: $.inputSchema.properties.fields.items)', + ); + } + return SemanticSnapshotCommand( + identifierPrefix: _nullableStringArg(args, 'identifierPrefix'), + subtreeOf: _nullableStringArg(args, 'subtreeOf'), + fields: fields, + ); + }, ), CommandSpec( name: 'tap_widget', diff --git a/mcp_server_dart/lib/src/skill_assets.g.dart b/mcp_server_dart/lib/src/skill_assets.g.dart index 692aa6d5..4b958df4 100644 --- a/mcp_server_dart/lib/src/skill_assets.g.dart +++ b/mcp_server_dart/lib/src/skill_assets.g.dart @@ -498,16 +498,25 @@ Returns: `{"extensionRPCs": ["ext.flutter.inspector.getRootWidget", "ext.mcp.too ### semantic_snapshot -Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. +Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. A node is listed when it reads (label, value), acts (button, text field, tap, scroll…) or carries a `Semantics(identifier:)`. +- `identifierPrefix` (string, optional) — keep only nodes whose identifier starts with it (`"nav."` for one rail). +- `subtreeOf` (string, optional) — keep one node and its descendants; a ref from the latest snapshot or an identifier, the ref tried first. +- `fields` (array of strings, optional) — node keys to return; `ref` is always kept. Names: `ref id type identifier label value hint enabled focused checked toggled selected bounds actions children visibleInViewport centerInViewport center`. - `connection` (object, optional) — connection override. +The tree is always walked whole, so a ref read off a filtered snapshot is the same ref the full snapshot would give and works with every interaction tool. A filtered reply adds `totalNodeCount` and echoes `filter`; `children` lists kept refs only. + ``` semantic_snapshot() +semantic_snapshot(identifierPrefix: "nav.", fields: ["identifier", "selected"]) +semantic_snapshot(subtreeOf: "panel.tabs") ``` -Returns: `{"snapshot_id": 3, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"]}]}` +Returns: `{"snapshot_id": 3, "nodeCount": 1, "viewport": {...}, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"], "bounds": {...}, "visibleInViewport": true, "centerInViewport": true, "center": {...}}]}` — the viewport is stated once in the envelope, not on each node. +- `subtree_root_not_found` — `subtreeOf` is neither a ref of the latest snapshot nor an identifier in the tree; no snapshot was taken, refs and `snapshot_id` are unchanged. +- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. - `vm_service_unavailable` — app not running or `MCPToolkitBinding.initialize()` not called. - `connection_selection_required` — multiple targets; supply `connection.targetId`. @@ -588,7 +597,7 @@ Use this skill when you need to drive a running Flutter app as a user would: ## Selectors -Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. +Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it — narrow a large screen with `identifierPrefix`, `subtreeOf` (a ref or an identifier) or `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. Snapshot node fields to filter on: diff --git a/mcp_server_dart/test/command_catalog_test.dart b/mcp_server_dart/test/command_catalog_test.dart index 7c245a29..ff375127 100644 --- a/mcp_server_dart/test/command_catalog_test.dart +++ b/mcp_server_dart/test/command_catalog_test.dart @@ -207,6 +207,33 @@ void main() { expect(command, isA()); }); + test('carries semantic_snapshot filters into the command', () { + final command = + catalog.buildCommand('semantic_snapshot', { + 'identifierPrefix': 'nav.', + 'subtreeOf': 's_3', + 'fields': ['identifier', 'label'], + }) + as SemanticSnapshotCommand; + expect(command.identifierPrefix, 'nav.'); + expect(command.subtreeOf, 's_3'); + expect(command.fields, ['identifier', 'label']); + }); + + test('rejects semantic_snapshot fields that are not an array', () { + expect( + () => catalog.buildCommand('semantic_snapshot', {'fields': 'label'}), + throwsA(anything), + ); + expect( + () => catalog.buildCommand('semantic_snapshot', { + 'fields': ['nope'], + }), + throwsA(isA()), + reason: 'the enum in the schema names the accepted fields', + ); + }); + test('reveal_search command is registered with bounded schema', () { final spec = catalog.specFor('reveal_search'); expect(spec, isNotNull); diff --git a/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart b/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart index c62a0b26..60c824ff 100644 --- a/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart +++ b/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart @@ -5,8 +5,43 @@ import 'dart:ui' as ui; import 'package:flutter/rendering.dart'; import 'package:flutter/widgets.dart'; +import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart' + show semanticSnapshotNodeFields; + import 'background_frame_pump.dart'; +/// What a `semantic_snapshot` call keeps of the full tree. +/// +/// The tree is always walked whole — refs number the full walk, so a ref +/// read off a filtered snapshot stays valid for every interaction tool. The +/// filter only decides which nodes, and which of their fields, are returned. +class SemanticSnapshotFilter { + const SemanticSnapshotFilter({ + this.identifierPrefix, + this.subtreeOf, + this.fields, + }); + + /// Keep nodes whose identifier starts with this prefix. + final String? identifierPrefix; + + /// Keep one node and its descendants: a ref from the latest snapshot or a + /// Semantics identifier, the ref tried first. + final String? subtreeOf; + + /// Node keys to return; `ref` is always kept. + final List? fields; + + bool get isEmpty => + identifierPrefix == null && subtreeOf == null && fields == null; + + Map toMap() => { + if (identifierPrefix != null) 'identifierPrefix': identifierPrefix, + if (subtreeOf != null) 'subtreeOf': subtreeOf, + if (fields != null) 'fields': fields, + }; +} + /// A service that walks the Flutter semantics tree and produces a compact, /// AI-friendly snapshot of interactive / meaningful elements. /// @@ -97,14 +132,20 @@ mixin SemanticSnapshotService { final bounds = resolveBounds(ref); final center = resolveCenter(ref); final viewport = viewportRect; - return visibilityForBounds( - bounds: bounds, - center: center, - viewport: viewport, - ); + return { + ...visibilityForBounds( + bounds: bounds, + center: center, + viewport: viewport, + ), + if (viewport != null) 'viewport': _rectToMap(viewport), + }; } /// Visibility metadata for logical bounds in the current Flutter viewport. + /// + /// The viewport itself is not part of the answer: a snapshot states it once + /// in its envelope, and a single-ref reply adds it in [visibilityForRef]. static Map visibilityForBounds({ required final ui.Rect? bounds, required final ui.Offset? center, @@ -122,7 +163,6 @@ mixin SemanticSnapshotService { 'visibleInViewport': visible, 'centerInViewport': centerVisible, if (bounds != null) 'bounds': _rectToMap(bounds), - if (viewport != null) 'viewport': _rectToMap(viewport), if (center != null) 'center': {'x': center.dx, 'y': center.dy}, }; @@ -172,8 +212,9 @@ mixin SemanticSnapshotService { /// Async so we can await a frame on the (rare) cold path where the /// semantics tree hasn't been primed yet — e.g. if /// [MCPToolkitBinding.initialize] didn't run for some reason. - static Future> buildSemanticSnapshot() => - _buildSnapshot(incrementId: true); + static Future> buildSemanticSnapshot({ + final SemanticSnapshotFilter? filter, + }) => _buildSnapshot(incrementId: true, filter: filter); /// Internal: read the snapshot without bumping the public id stream. /// Used by `WaitPredicateService` while polling so callers' outstanding @@ -245,7 +286,24 @@ mixin SemanticSnapshotService { static Future> _buildSnapshot({ required final bool incrementId, + final SemanticSnapshotFilter? filter, }) async { + final unknownFields = filter?.fields + ?.where((final f) => !semanticSnapshotNodeFields.contains(f)) + .toList(); + if (unknownFields != null && unknownFields.isNotEmpty) { + return { + 'success': false, + 'error': 'unknown_field', + 'unknownFields': unknownFields, + 'acceptedFields': semanticSnapshotNodeFields, + 'hint': + 'fields names keys of a snapshot node, and ' + '${unknownFields.map((final f) => '"$f"').join(', ')} ' + '${unknownFields.length == 1 ? 'is' : 'are'} not among them — ' + 'pick from acceptedFields. No snapshot was taken.', + }; + } final binding = WidgetsBinding.instance; final isTestBinding = _isInFlutterTest(); @@ -279,7 +337,7 @@ mixin SemanticSnapshotService { await pumpFramesIfSuspended(); } - return await _buildSnapshotBody(incrementId: incrementId); + return await _buildSnapshotBody(incrementId: incrementId, filter: filter); } finally { if (isTestBinding) { handle.dispose(); @@ -296,11 +354,36 @@ mixin SemanticSnapshotService { static Future> _buildSnapshotBody({ required final bool incrementId, + final SemanticSnapshotFilter? filter, }) async { final refMap = {}; final boundsMap = {}; final centerMap = {}; + // Resolved before the counter moves and the ref maps are replaced, so a + // refusal leaves the caller's refs and snapshot id exactly as they were. + final subtreeOf = filter?.subtreeOf; + SemanticsNode? subtreeRoot; + if (subtreeOf != null) { + subtreeRoot = + _lastRefMap[subtreeOf] ?? + _findByIdentifier(_currentRootNode(), subtreeOf); + if (subtreeRoot == null) { + return { + 'success': false, + 'error': 'subtree_root_not_found', + 'subtreeOf': subtreeOf, + 'snapshotId': _snapshotCounter, + 'hint': + '"$subtreeOf" is neither a ref of snapshot $_snapshotCounter ' + 'nor an identifier the tree publishes. Refs expire with every ' + 'snapshot, so pass an identifier when one exists, or take an ' + 'unfiltered snapshot and use a ref from it. No snapshot was ' + 'taken.', + }; + } + } + final snapshotId = incrementId ? ++_snapshotCounter : _snapshotCounter; SemanticsNode? root; @@ -446,16 +529,112 @@ mixin SemanticSnapshotService { ); } + final totalNodeCount = nodes.length; + final returned = filter == null || filter.isEmpty + ? nodes + : _applyFilter( + nodes: nodes, + refMap: refMap, + filter: filter, + subtreeRoot: subtreeRoot, + ); + return { 'snapshot_id': snapshotId, - 'nodes': nodes, - 'nodeCount': nodes.length, + 'nodes': returned, + 'nodeCount': returned.length, + if (filter != null && !filter.isEmpty) ...{ + 'totalNodeCount': totalNodeCount, + 'filter': filter.toMap(), + }, 'truncated': truncated, - 'interactionSurface': _classifyInteractionSurface(nodes.length), + // The surface describes the app, not the slice asked for. + 'interactionSurface': _classifyInteractionSurface(totalNodeCount), if (viewport != null) 'viewport': _rectToMap(viewport), }; } + /// The nodes a [filter] keeps, with `children` pruned to kept refs and + /// fields projected to the requested set. + static List> _applyFilter({ + required final List> nodes, + required final Map refMap, + required final SemanticSnapshotFilter filter, + required final SemanticsNode? subtreeRoot, + }) { + final prefix = filter.identifierPrefix; + final kept = >[ + for (final node in nodes) + if ((subtreeRoot == null || + _isWithin(refMap[node['ref']! as String], subtreeRoot)) && + (prefix == null || + (node['identifier'] is String && + (node['identifier']! as String).startsWith(prefix)))) + node, + ]; + final keptRefs = {for (final node in kept) node['ref']}; + final fields = filter.fields?.toSet(); + return >[ + for (final node in kept) + { + for (final MapEntry(:key, :value) in node.entries) + if (fields == null || key == 'ref' || fields.contains(key)) + if (key == 'children') + key: [ + for (final child in value! as List) + if (keptRefs.contains(child)) child, + ] + else + key: value, + }..removeWhere( + (final key, final value) => + key == 'children' && (value! as List).isEmpty, + ), + ]; + } + + /// Whether [node] is [root] or sits below it. + static bool _isWithin(final SemanticsNode? node, final SemanticsNode root) { + for (var current = node; current != null; current = current.parent) { + if (identical(current, root)) return true; + } + return false; + } + + static SemanticsNode? _currentRootNode() { + try { + final renderViews = WidgetsBinding.instance.renderViews; + if (renderViews.isEmpty) return null; + final owner = (_activeRenderView ?? renderViews.first).owner; + return owner?.semanticsOwner?.rootSemanticsNode; + } on Exception { + // The body reports the missing tree itself; the lookup just found nothing. + return null; + } + } + + static SemanticsNode? _findByIdentifier( + final SemanticsNode? root, + final String identifier, + ) { + if (root == null) return null; + SemanticsNode? found; + void walk(final SemanticsNode node) { + if (found != null) return; + if (node.getSemanticsData().identifier == identifier) { + found = node; + return; + } + node.visitChildren((final child) { + walk(child); + return found == null; + }); + } + + walk(root); + return found; + } + /// How agents should interact with this app's visible surface. /// /// - [flutter_widgets]: semantics expose tappable refs (normal Material/Cupertino). @@ -543,6 +722,11 @@ mixin SemanticSnapshotService { // Has a semantic label or value worth surfacing. if (data.label.isNotEmpty || data.value.isNotEmpty) return true; + // An identifier is a handle the app published on purpose — a container + // named "rail" or "panel" exists to be addressed, by subtreeOf or by + // wait_for, even though it neither reads nor acts. + if (data.identifier.isNotEmpty) return true; + // Interactive flags. if (data.hasFlag(SemanticsFlag.isButton)) return true; if (data.hasFlag(SemanticsFlag.isTextField)) return true; diff --git a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart index 07c1e629..c6eb7c44 100644 --- a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart +++ b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart @@ -92,10 +92,21 @@ extension type OnSemanticSnapshotEntry._(AgentCallEntry entry) factory OnSemanticSnapshotEntry() { final entry = mcpToolkitTool( handler: (final parameters) async { - final snapshot = await SemanticSnapshotService.buildSemanticSnapshot(); + final rawFields = parameters['fields'] as Object?; + final filter = SemanticSnapshotFilter( + identifierPrefix: _nonEmptyString(parameters['identifierPrefix']), + subtreeOf: _nonEmptyString(parameters['subtreeOf']), + fields: rawFields is List + ? rawFields.map((final f) => f.toString()).toList() + : null, + ); + final snapshot = await SemanticSnapshotService.buildSemanticSnapshot( + filter: filter.isEmpty ? null : filter, + ); return MCPCallResult( - message: - 'Semantic snapshot captured. Use refs to interact with widgets.', + message: snapshot['success'] == false + ? 'Semantic snapshot refused: ${snapshot['error']}' + : 'Semantic snapshot captured. Use refs to interact with widgets.', parameters: snapshot, ); }, @@ -104,6 +115,8 @@ extension type OnSemanticSnapshotEntry._(AgentCallEntry entry) description: 'Get compact semantic tree of interactive widgets with refs ' 'for interaction tools (tap_widget, enter_text, etc.). ' + 'Narrow it with identifierPrefix, subtreeOf (a ref or an ' + 'identifier) and fields; refs are those of the full tree. ' 'A control that appears only under the pointer is absent here: ' 'on desktop and web, hover the element that should own it and ' 'snapshot again.', @@ -871,3 +884,9 @@ extension type OnFocusWidgetEntry._(AgentCallEntry entry) return OnFocusWidgetEntry._(entry); } } + +String? _nonEmptyString(final Object? value) { + if (value is! String) return null; + final trimmed = value.trim(); + return trimmed.isEmpty ? null : trimmed; +} diff --git a/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart b/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart index 7a1d39a5..ae41a97f 100644 --- a/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart +++ b/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart @@ -55,6 +55,12 @@ void main() { expect(schema.containsKey('required'), isFalse); final props = schema['properties']! as Map; expect(props.containsKey('connection'), isTrue); + expect(props.containsKey('identifierPrefix'), isTrue); + expect(props.containsKey('subtreeOf'), isTrue); + final fields = props['fields']! as Map; + final items = fields['items']! as Map; + expect(items['enum'], semanticSnapshotNodeFields); + expect(semanticSnapshotNodeFields, contains('ref')); }); test('wait_for', () { diff --git a/mcp_toolkit/test/semantic_snapshot_surface_test.dart b/mcp_toolkit/test/semantic_snapshot_surface_test.dart index 4dcaddca..37614635 100644 --- a/mcp_toolkit/test/semantic_snapshot_surface_test.dart +++ b/mcp_toolkit/test/semantic_snapshot_surface_test.dart @@ -1,4 +1,6 @@ import 'package:flutter/material.dart'; +import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart' + show semanticSnapshotNodeFields; import 'package:flutter_test/flutter_test.dart'; import 'package:mcp_toolkit/mcp_toolkit.dart'; @@ -461,6 +463,256 @@ void main() { }, ); + group('semantic_snapshot filters', () { + Future pumpRailAndPanel(final WidgetTester tester) async { + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: Row( + children: [ + Semantics( + identifier: 'rail', + container: true, + child: Column( + children: [ + Semantics( + identifier: 'nav.tasks', + button: true, + onTap: () {}, + child: const Text('Tasks'), + ), + Semantics( + identifier: 'nav.calendar', + button: true, + onTap: () {}, + child: const Text('Calendar'), + ), + ], + ), + ), + Semantics( + identifier: 'panel', + container: true, + child: Column( + children: [ + Semantics( + identifier: 'panel.tab.overview', + button: true, + onTap: () {}, + child: const Text('Overview'), + ), + // No identifier and no boundary of its own: the label + // still has to travel with the button node. + Semantics( + container: true, + button: true, + onTap: () {}, + child: const Text('Unnamed button'), + ), + ], + ), + ), + ], + ), + ), + ), + ); + await tester.pumpAndSettle(); + } + + List> nodesOf(final Map snapshot) => + (snapshot['nodes']! as List).cast>(); + + testWidgets('the viewport is stated once, not on every node', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final snapshot = await _snapshotAfterPump(tester); + + expect(snapshot['viewport'], isA>()); + for (final node in nodesOf(snapshot)) { + expect( + node.containsKey('viewport'), + isFalse, + reason: '${node['ref']}', + ); + expect(node.containsKey('visibleInViewport'), isTrue); + } + // A single-ref answer still carries it, since it has no envelope. + final ref = nodesOf(snapshot).first['ref']! as String; + expect( + SemanticSnapshotService.visibilityForRef(ref)['viewport'], + isA>(), + ); + } finally { + semantics.dispose(); + } + }); + + testWidgets('identifierPrefix keeps the matching nodes only', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final full = await _snapshotAfterPump(tester); + final fullRefs = { + for (final node in nodesOf(full)) + if (node['identifier'] is String) + node['identifier']! as String: node['ref']! as String, + }; + + final future = SemanticSnapshotService.buildSemanticSnapshot( + filter: const SemanticSnapshotFilter(identifierPrefix: 'nav.'), + ); + await tester.pump(); + final filtered = await future; + + final nodes = nodesOf(filtered); + expect( + nodes.map((final n) => n['identifier']), + unorderedEquals(['nav.tasks', 'nav.calendar']), + ); + // Refs number the full walk, so they agree with the unfiltered + // snapshot and every interaction tool can resolve them. + for (final node in nodes) { + expect(node['ref'], fullRefs[node['identifier']]); + expect( + SemanticSnapshotService.resolveRef(node['ref']! as String), + isNotNull, + ); + } + expect(filtered['nodeCount'], 2); + expect(filtered['totalNodeCount'], nodesOf(full).length); + expect(filtered['filter'], { + 'identifierPrefix': 'nav.', + }); + expect(filtered['interactionSurface'], full['interactionSurface']); + } finally { + semantics.dispose(); + } + }); + + testWidgets('subtreeOf narrows to a node and its descendants', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + + // By identifier: no earlier snapshot is needed. + var future = SemanticSnapshotService.buildSemanticSnapshot( + filter: const SemanticSnapshotFilter(subtreeOf: 'panel'), + ); + await tester.pump(); + final byIdentifier = await future; + final panelNodes = nodesOf(byIdentifier); + expect( + panelNodes.map((final n) => n['identifier'] ?? n['label']), + unorderedEquals([ + 'panel', + 'panel.tab.overview', + 'Unnamed button', + ]), + ); + // Children of the kept container name kept refs only. + final panel = panelNodes.firstWhere( + (final n) => n['identifier'] == 'panel', + ); + final keptRefs = panelNodes.map((final n) => n['ref']).toSet(); + expect( + keptRefs.containsAll(panel['children']! as List), + isTrue, + ); + + // By ref from the latest snapshot. + final railRef = + nodesOf( + await _snapshotAfterPump(tester), + ).firstWhere((final n) => n['identifier'] == 'rail')['ref']! + as String; + future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(subtreeOf: railRef), + ); + await tester.pump(); + final byRef = await future; + expect( + nodesOf(byRef).map((final n) => n['identifier']), + unorderedEquals(['rail', 'nav.tasks', 'nav.calendar']), + ); + } finally { + semantics.dispose(); + } + }); + + testWidgets( + 'an unknown subtree root is refused before a snapshot is spent', + (final tester) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final before = await _snapshotAfterPump(tester); + final knownRef = nodesOf(before).first['ref']! as String; + + final future = SemanticSnapshotService.buildSemanticSnapshot( + filter: const SemanticSnapshotFilter(subtreeOf: 'no.such.node'), + ); + await tester.pump(); + final result = await future; + + expect(result['success'], isFalse); + expect(result['error'], 'subtree_root_not_found'); + expect(result['hint'], contains('No snapshot was taken')); + expect( + SemanticSnapshotService.currentSnapshotId, + before['snapshot_id'], + reason: 'a refusal must not move the counter', + ); + expect(SemanticSnapshotService.resolveRef(knownRef), isNotNull); + } finally { + semantics.dispose(); + } + }, + ); + + testWidgets('fields projects every node and always keeps the ref', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + + var future = SemanticSnapshotService.buildSemanticSnapshot( + filter: const SemanticSnapshotFilter( + fields: ['identifier', 'label'], + ), + ); + await tester.pump(); + final projected = await future; + for (final node in nodesOf(projected)) { + expect(node.keys, isNot(contains('bounds'))); + expect(node.keys, isNot(contains('actions'))); + expect(node.keys, contains('ref')); + expect(node.keys, anyOf(contains('identifier'), contains('label'))); + } + + future = SemanticSnapshotService.buildSemanticSnapshot( + filter: const SemanticSnapshotFilter(fields: ['ref', 'nope']), + ); + await tester.pump(); + final refused = await future; + expect(refused['success'], isFalse); + expect(refused['error'], 'unknown_field'); + expect(refused['unknownFields'], ['nope']); + expect(refused['acceptedFields'], semanticSnapshotNodeFields); + expect(refused['hint'], contains('"nope"')); + } finally { + semantics.dispose(); + } + }); + }); testWidgets( 'semantic_snapshot reports hybrid when no interactive semantics refs', (final tester) async { diff --git a/packages/core/lib/src/commands/core_commands.dart b/packages/core/lib/src/commands/core_commands.dart index ebf5ac45..a8997079 100644 --- a/packages/core/lib/src/commands/core_commands.dart +++ b/packages/core/lib/src/commands/core_commands.dart @@ -295,7 +295,15 @@ final class CaptureUiSnapshotCommand extends CoreCommand { } final class SemanticSnapshotCommand extends CoreCommand { - const SemanticSnapshotCommand(); + const SemanticSnapshotCommand({ + this.identifierPrefix, + this.subtreeOf, + this.fields, + }); + + final String? identifierPrefix; + final String? subtreeOf; + final List? fields; @override String get name => 'semantic_snapshot'; diff --git a/packages/core/lib/src/tools/interaction_input_schemas.dart b/packages/core/lib/src/tools/interaction_input_schemas.dart index d664bea2..56a9d1e9 100644 --- a/packages/core/lib/src/tools/interaction_input_schemas.dart +++ b/packages/core/lib/src/tools/interaction_input_schemas.dart @@ -44,11 +44,63 @@ Map tapWidgetInputSchema() => { }, }; +/// Every key a `semantic_snapshot` node can carry, in the order the node +/// publishes them. `fields` selects among these; `ref` is always returned. +const List semanticSnapshotNodeFields = [ + 'ref', + 'id', + 'type', + 'identifier', + 'label', + 'value', + 'hint', + 'enabled', + 'focused', + 'checked', + 'toggled', + 'selected', + 'bounds', + 'actions', + 'children', + 'visibleInViewport', + 'centerInViewport', + 'center', +]; + /// Shared JSON Schema for [semantic_snapshot] / `fmt_semantic_snapshot`. Map semanticSnapshotInputSchema() => { 'type': 'object', 'additionalProperties': false, - 'properties': {'connection': connectionOverrideJsonSchema()}, + 'properties': { + 'identifierPrefix': { + 'type': 'string', + 'description': + 'Return only nodes whose Semantics identifier starts with this ' + 'prefix ("nav." for one navigation rail). Refs are those of the ' + 'full tree, so a ref from a filtered snapshot works everywhere.', + }, + 'subtreeOf': { + 'type': 'string', + 'description': + 'Return only this node and its descendants. A ref from the latest ' + 'snapshot ("s_12") or a Semantics identifier; a ref is tried ' + 'first. Fails with subtree_root_not_found without spending a ' + 'snapshot when neither resolves.', + }, + 'fields': { + 'type': 'array', + 'items': { + 'type': 'string', + 'enum': semanticSnapshotNodeFields, + }, + 'minItems': 1, + 'uniqueItems': true, + 'description': + 'Node fields to return; "ref" is always included. Omit for every ' + 'field. A name outside the list fails with unknown_field.', + }, + 'connection': connectionOverrideJsonSchema(), + }, }; /// Shared JSON Schema for [wait_for] / `fmt_wait_for`. diff --git a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md index f1284d6b..88a07c40 100644 --- a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md @@ -19,7 +19,7 @@ Use this skill when you need to drive a running Flutter app as a user would: ## Selectors -Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. +Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it — narrow a large screen with `identifierPrefix`, `subtreeOf` (a ref or an identifier) or `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. Snapshot node fields to filter on: diff --git a/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md b/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md index e199ef2f..d303a83d 100644 --- a/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md @@ -159,16 +159,25 @@ Returns: `{"extensionRPCs": ["ext.flutter.inspector.getRootWidget", "ext.mcp.too ### semantic_snapshot -Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. +Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. A node is listed when it reads (label, value), acts (button, text field, tap, scroll…) or carries a `Semantics(identifier:)`. +- `identifierPrefix` (string, optional) — keep only nodes whose identifier starts with it (`"nav."` for one rail). +- `subtreeOf` (string, optional) — keep one node and its descendants; a ref from the latest snapshot or an identifier, the ref tried first. +- `fields` (array of strings, optional) — node keys to return; `ref` is always kept. Names: `ref id type identifier label value hint enabled focused checked toggled selected bounds actions children visibleInViewport centerInViewport center`. - `connection` (object, optional) — connection override. +The tree is always walked whole, so a ref read off a filtered snapshot is the same ref the full snapshot would give and works with every interaction tool. A filtered reply adds `totalNodeCount` and echoes `filter`; `children` lists kept refs only. + ``` semantic_snapshot() +semantic_snapshot(identifierPrefix: "nav.", fields: ["identifier", "selected"]) +semantic_snapshot(subtreeOf: "panel.tabs") ``` -Returns: `{"snapshot_id": 3, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"]}]}` +Returns: `{"snapshot_id": 3, "nodeCount": 1, "viewport": {...}, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"], "bounds": {...}, "visibleInViewport": true, "centerInViewport": true, "center": {...}}]}` — the viewport is stated once in the envelope, not on each node. +- `subtree_root_not_found` — `subtreeOf` is neither a ref of the latest snapshot nor an identifier in the tree; no snapshot was taken, refs and `snapshot_id` are unchanged. +- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. - `vm_service_unavailable` — app not running or `MCPToolkitBinding.initialize()` not called. - `connection_selection_required` — multiple targets; supply `connection.targetId`. From 7b2aa3db9a61e7f07abdaa52885de2e2bacab6e7 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Sun, 6 Sep 2026 23:31:12 +0300 Subject: [PATCH 2/6] fix(semantic_snapshot): carry the filters over the MCP path and encode fields as JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCP tool built a bare SemanticSnapshotCommand, so identifierPrefix, subtreeOf and fields never left the host — only the CLI path, which goes through the command catalog, forwarded them. And the fields list reached the app as List.toString(), which the extension's schema coercion refuses because a list argument travels as JSON, the way wait_for sends its predicate. Two tests in server_capability_core pin the mapping. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Fao9J2ibPNtCFhmMRjXv96 --- .../lib/src/shared_core/command_executor.dart | 4 ++- .../lib/src/tools/semantic_tools.dart | 13 +++++++- .../test/tools/semantic_tools_test.dart | 31 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/mcp_server_dart/lib/src/shared_core/command_executor.dart b/mcp_server_dart/lib/src/shared_core/command_executor.dart index a0e09510..471166ab 100644 --- a/mcp_server_dart/lib/src/shared_core/command_executor.dart +++ b/mcp_server_dart/lib/src/shared_core/command_executor.dart @@ -943,7 +943,9 @@ final class DefaultCoreCommandExecutor implements CoreCommandExecutor { if (command.identifierPrefix != null) 'identifierPrefix': command.identifierPrefix, if (command.subtreeOf != null) 'subtreeOf': command.subtreeOf, - if (command.fields != null) 'fields': command.fields, + // Extension parameters travel as strings; the app decodes a + // list from JSON, not from List.toString(). + if (command.fields != null) 'fields': jsonEncode(command.fields), }, ); return CoreResult.success(data: _map(result.json)); diff --git a/packages/server_capability_core/lib/src/tools/semantic_tools.dart b/packages/server_capability_core/lib/src/tools/semantic_tools.dart index 26402cfb..017bdc3c 100644 --- a/packages/server_capability_core/lib/src/tools/semantic_tools.dart +++ b/packages/server_capability_core/lib/src/tools/semantic_tools.dart @@ -23,7 +23,18 @@ void registerSemanticTools(final CapabilityContext context) { 'snapshot again.', inputSchema: semanticSnapshotInputSchema(), handler: (final args) async { - return runCommand(runner, args, const SemanticSnapshotCommand()); + final rawFields = args['fields']; + return runCommand( + runner, + args, + SemanticSnapshotCommand( + identifierPrefix: stringArgOrNull(args['identifierPrefix']), + subtreeOf: stringArgOrNull(args['subtreeOf']), + fields: rawFields is List + ? rawFields.map((final f) => f.toString()).toList() + : null, + ), + ); }, ), ); diff --git a/packages/server_capability_core/test/tools/semantic_tools_test.dart b/packages/server_capability_core/test/tools/semantic_tools_test.dart index de1e0169..2dd68769 100644 --- a/packages/server_capability_core/test/tools/semantic_tools_test.dart +++ b/packages/server_capability_core/test/tools/semantic_tools_test.dart @@ -86,6 +86,37 @@ void main() { }, ); + test( + 'semantic_snapshot handler carries the filters into the command', + () async { + final runner = FakeCommandRunner() + ..nextExecuteResult = CoreResult.success(data: {}); + final ctx = _registeredCtx(runner: runner); + final reg = ctx.registrationFor('semantic_snapshot')!; + await reg.handler(const { + 'identifierPrefix': 'nav.', + 'subtreeOf': 'panel', + 'fields': ['identifier', 'selected'], + }); + final cmd = runner.executedCommands.single as SemanticSnapshotCommand; + expect(cmd.identifierPrefix, 'nav.'); + expect(cmd.subtreeOf, 'panel'); + expect(cmd.fields, ['identifier', 'selected']); + }, + ); + + test('semantic_snapshot handler leaves absent filters null', () async { + final runner = FakeCommandRunner() + ..nextExecuteResult = CoreResult.success(data: {}); + final ctx = _registeredCtx(runner: runner); + final reg = ctx.registrationFor('semantic_snapshot')!; + await reg.handler(const {'identifierPrefix': ' '}); + final cmd = runner.executedCommands.single as SemanticSnapshotCommand; + expect(cmd.identifierPrefix, isNull); + expect(cmd.subtreeOf, isNull); + expect(cmd.fields, isNull); + }); + test( 'semantic_snapshot handler calls applyConnectionOverride before execute', () async { From 10cee4ef67fefa92411ede9f6c1e7fef00af6824 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Sun, 6 Sep 2026 23:34:06 +0300 Subject: [PATCH 3/6] fix(semantic_snapshot): read the fields list off the wire map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A legacy handler receives every argument as a string — the toolkit wrapper re-encodes lists as JSON — so the entry saw text where it looked for a List and dropped the projection silently. The entry now decodes the JSON it is handed; a test drives the entry through invokeDirect, the same path the service extension takes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Fao9J2ibPNtCFhmMRjXv96 --- .../lib/src/toolkits/interaction_toolkit.dart | 20 +++++++++--- .../test/semantic_snapshot_surface_test.dart | 31 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart index c6eb7c44..3dd258f9 100644 --- a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart +++ b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:dart_mcp/client.dart'; import 'package:flutter/gestures.dart' show PointerDeviceKind; import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart'; @@ -92,13 +94,10 @@ extension type OnSemanticSnapshotEntry._(AgentCallEntry entry) factory OnSemanticSnapshotEntry() { final entry = mcpToolkitTool( handler: (final parameters) async { - final rawFields = parameters['fields'] as Object?; final filter = SemanticSnapshotFilter( identifierPrefix: _nonEmptyString(parameters['identifierPrefix']), subtreeOf: _nonEmptyString(parameters['subtreeOf']), - fields: rawFields is List - ? rawFields.map((final f) => f.toString()).toList() - : null, + fields: _stringList(parameters['fields']), ); final snapshot = await SemanticSnapshotService.buildSemanticSnapshot( filter: filter.isEmpty ? null : filter, @@ -885,6 +884,19 @@ extension type OnFocusWidgetEntry._(AgentCallEntry entry) } } +/// A list argument reaches a legacy handler JSON-encoded — the wire map holds +/// strings — and already validated against the schema, so the text is JSON. +List? _stringList(final Object? value) { + final decoded = switch (value) { + final String text when text.trim().isNotEmpty => jsonDecode(text), + final List list => list, + _ => null, + }; + return decoded is List + ? decoded.map((final f) => f.toString()).toList() + : null; +} + String? _nonEmptyString(final Object? value) { if (value is! String) return null; final trimmed = value.trim(); diff --git a/mcp_toolkit/test/semantic_snapshot_surface_test.dart b/mcp_toolkit/test/semantic_snapshot_surface_test.dart index 37614635..3edba952 100644 --- a/mcp_toolkit/test/semantic_snapshot_surface_test.dart +++ b/mcp_toolkit/test/semantic_snapshot_surface_test.dart @@ -677,6 +677,37 @@ void main() { }, ); + testWidgets('the entry decodes fields from the wire map', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + // The service extension hands a legacy handler strings only, so the + // list arrives JSON-encoded; invokeDirect walks that same path. + final future = OnSemanticSnapshotEntry().invokeDirect({ + 'identifierPrefix': 'nav.', + 'fields': ['identifier', 'label'], + }); + await tester.pump(); + final result = await future; + + expect(result.ok, isTrue, reason: result.message); + expect(result.data['filter'], { + 'identifierPrefix': 'nav.', + 'fields': ['identifier', 'label'], + }); + for (final node in nodesOf(result.data)) { + expect( + node.keys, + unorderedEquals(['ref', 'identifier', 'label']), + ); + } + } finally { + semantics.dispose(); + } + }); + testWidgets('fields projects every node and always keeps the ref', ( final tester, ) async { From 4ef98e9635a757d2c07ca2baaf2b65f663bb614d Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Tue, 8 Sep 2026 10:16:32 +0300 Subject: [PATCH 4/6] fix(semantic_snapshot): fail the command when the snapshot was refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A snapshot payload carries `success` only when it refused, so the executor reported `subtree_root_not_found` as a successful command: the caller read an empty payload as a screen that went empty, while the refs it already held were still current — no snapshot had been taken. The refusal now routes to a failure that keeps the payload, alongside the routing the other interaction tools use. A captured snapshot has no `success` key at all, so the absent key stays a success. `fields` is validated at the MCP/CLI boundary, before a call reaches the app, so the skill and the tool schema describe that refusal instead of promising an `unknown_field` reply no caller on that path can see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Pq6wQ3gULA8JMPGdH2HxGj --- .../lib/src/shared_core/command_executor.dart | 18 ++++++++++- mcp_server_dart/lib/src/skill_assets.g.dart | 2 +- .../interaction_response_routing_test.dart | 30 +++++++++++++++++++ .../src/tools/interaction_input_schemas.dart | 3 +- .../flutter-mcp-toolkit-inspect/SKILL.md | 2 +- 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/mcp_server_dart/lib/src/shared_core/command_executor.dart b/mcp_server_dart/lib/src/shared_core/command_executor.dart index 471166ab..9a522bd0 100644 --- a/mcp_server_dart/lib/src/shared_core/command_executor.dart +++ b/mcp_server_dart/lib/src/shared_core/command_executor.dart @@ -948,7 +948,7 @@ final class DefaultCoreCommandExecutor implements CoreCommandExecutor { if (command.fields != null) 'fields': jsonEncode(command.fields), }, ); - return CoreResult.success(data: _map(result.json)); + return routeSemanticSnapshotResponse(_map(result.json)); } on Exception catch (e) { return CoreResult.failure( code: CoreErrorCode.semanticSnapshotFailed, @@ -2037,6 +2037,22 @@ CoreResult routeInteractionResponse( ); } +/// Route a `semantic_snapshot` payload to a result that matches its verdict. +/// +/// A captured snapshot carries no `success` key — only a refusal does, so an +/// absent key is a success. A filter that names an unresolvable `subtreeOf` +/// takes no snapshot at all, and reaching the caller as a successful command +/// would let it read stale refs as if they had just been issued. +CoreResult routeSemanticSnapshotResponse(final Map data) { + if (data['success'] != false) return CoreResult.success(data: data); + return CoreResult.failure( + code: CoreErrorCode.semanticSnapshotFailed, + message: + 'semantic_snapshot refused: ${data['error'] ?? 'no reason given'}', + details: data, + ); +} + /// Route the toolkit's `wait_for` extension response to a [CoreResult]. /// /// [data] is the decoded toolkit response. Returns success for diff --git a/mcp_server_dart/lib/src/skill_assets.g.dart b/mcp_server_dart/lib/src/skill_assets.g.dart index 4b958df4..7660ffef 100644 --- a/mcp_server_dart/lib/src/skill_assets.g.dart +++ b/mcp_server_dart/lib/src/skill_assets.g.dart @@ -516,7 +516,7 @@ semantic_snapshot(subtreeOf: "panel.tabs") Returns: `{"snapshot_id": 3, "nodeCount": 1, "viewport": {...}, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"], "bounds": {...}, "visibleInViewport": true, "centerInViewport": true, "center": {...}}]}` — the viewport is stated once in the envelope, not on each node. - `subtree_root_not_found` — `subtreeOf` is neither a ref of the latest snapshot nor an identifier in the tree; no snapshot was taken, refs and `snapshot_id` are unchanged. -- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. +- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. Over MCP and the CLI the name is refused at the boundary instead — an invalid `fields` argument naming the accepted keys, without spending a call on the app. - `vm_service_unavailable` — app not running or `MCPToolkitBinding.initialize()` not called. - `connection_selection_required` — multiple targets; supply `connection.targetId`. diff --git a/mcp_server_dart/test/interaction_response_routing_test.dart b/mcp_server_dart/test/interaction_response_routing_test.dart index 18aba698..1e00682a 100644 --- a/mcp_server_dart/test/interaction_response_routing_test.dart +++ b/mcp_server_dart/test/interaction_response_routing_test.dart @@ -83,4 +83,34 @@ void main() { expect(recovery.containsKey('fix_command'), isFalse); }); }); + + group('routeSemanticSnapshotResponse', () { + test('a captured snapshot has no success key and still routes to ok', () { + final result = routeSemanticSnapshotResponse({ + 'snapshot_id': 7, + 'nodes': [ + {'ref': 's_0', 'identifier': 'nav.tasks'}, + ], + }); + + expect(result.ok, isTrue); + expect((result.data! as Map)['snapshot_id'], 7); + }); + + test('an unresolvable subtreeOf is a failure, not an empty snapshot', () { + // No snapshot was taken, so the refs the caller already holds stay + // current — reporting success would read as a screen that went empty. + final result = routeSemanticSnapshotResponse({ + 'success': false, + 'error': 'subtree_root_not_found', + 'subtreeOf': 'panel.missing', + 'hint': 'Take a full snapshot and read the identifier off it.', + }); + + expect(result.ok, isFalse); + expect(result.error!.code, CoreErrorCode.semanticSnapshotFailed); + expect(result.error!.message, contains('subtree_root_not_found')); + expect((result.error!.details! as Map)['subtreeOf'], 'panel.missing'); + }); + }); } diff --git a/packages/core/lib/src/tools/interaction_input_schemas.dart b/packages/core/lib/src/tools/interaction_input_schemas.dart index 56a9d1e9..24147ae4 100644 --- a/packages/core/lib/src/tools/interaction_input_schemas.dart +++ b/packages/core/lib/src/tools/interaction_input_schemas.dart @@ -97,7 +97,8 @@ Map semanticSnapshotInputSchema() => { 'uniqueItems': true, 'description': 'Node fields to return; "ref" is always included. Omit for every ' - 'field. A name outside the list fails with unknown_field.', + 'field. A name outside the list is refused before the call reaches ' + 'the app.', }, 'connection': connectionOverrideJsonSchema(), }, diff --git a/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md b/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md index d303a83d..6565741a 100644 --- a/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md @@ -177,7 +177,7 @@ semantic_snapshot(subtreeOf: "panel.tabs") Returns: `{"snapshot_id": 3, "nodeCount": 1, "viewport": {...}, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"], "bounds": {...}, "visibleInViewport": true, "centerInViewport": true, "center": {...}}]}` — the viewport is stated once in the envelope, not on each node. - `subtree_root_not_found` — `subtreeOf` is neither a ref of the latest snapshot nor an identifier in the tree; no snapshot was taken, refs and `snapshot_id` are unchanged. -- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. +- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. Over MCP and the CLI the name is refused at the boundary instead — an invalid `fields` argument naming the accepted keys, without spending a call on the app. - `vm_service_unavailable` — app not running or `MCPToolkitBinding.initialize()` not called. - `connection_selection_required` — multiple targets; supply `connection.targetId`. From 22ac9d7b2d9a31e98b6c8ca12932e3bb3693e500 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Tue, 8 Sep 2026 11:03:42 +0300 Subject: [PATCH 5/6] refactor(semantic_snapshot): answer the filter review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SemanticSnapshotFilter` becomes a const extension type over the map the snapshot echoes back as `filter`, with `fromJson`/`toJson` and the `from_json_to_json` decoders the repository's models use. It reads the wire itself, so the two ad-hoc helpers in the toolkit entry are gone: a list argument arriving as JSON text decodes like a list, and a selector is kept as the caller wrote it — identifiers are matched whole, so a trimmed one selects a different node. A `subtreeOf` ref whose node has left the tree is now refused. The ref map still pointed at the detached `SemanticsNode`, no live node filtered into it, and an expired ref came back as a screen that had gone blank instead of the `subtree_root_not_found` it is. Tightening the catalog test to `isA()` exposed the third: `fields: "label"` threw a `FormatException` out of `jsonDecode` inside schema coercion, past every caller that answers invalid arguments. The gateway now reports it the way it reports a validation failure. The control skill listed `tooltip`, `key` and `flags` among the keys to scan, and a snapshot node carries none of them; it also read as if `fields` narrowed the node set, which is what `identifierPrefix` and `subtreeOf` do. --- .../dynamic_registry/dynamic_gateway.dart | 9 +++ mcp_server_dart/lib/src/skill_assets.g.dart | 9 +-- .../test/command_catalog_test.dart | 2 +- .../services/semantic_snapshot_service.dart | 78 +++++++++++++++---- .../lib/src/toolkits/interaction_toolkit.dart | 27 +------ .../test/semantic_snapshot_surface_test.dart | 49 ++++++++++-- .../flutter-mcp-toolkit-control/SKILL.md | 9 +-- 7 files changed, 124 insertions(+), 59 deletions(-) diff --git a/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart b/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart index 195389ba..a48efe3e 100644 --- a/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart +++ b/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart @@ -427,6 +427,15 @@ CoreResult? validationFailureForDynamicSchema({ code: CoreErrorCode.invalidCommand, message: e.message, ); + } on FormatException catch (e) { + // An array or object argument travels as wire text and is decoded while + // coercing. Text that is not JSON throws out of the decoder itself, and + // one mistyped argument must read back as an invalid command, not as a + // crash from inside validation. + return CoreResult.failure( + code: CoreErrorCode.invalidCommand, + message: 'Invalid argument for $subjectLabel: ${e.message}', + ); } } diff --git a/mcp_server_dart/lib/src/skill_assets.g.dart b/mcp_server_dart/lib/src/skill_assets.g.dart index d5a96899..bef5a600 100644 --- a/mcp_server_dart/lib/src/skill_assets.g.dart +++ b/mcp_server_dart/lib/src/skill_assets.g.dart @@ -597,17 +597,16 @@ Use this skill when you need to drive a running Flutter app as a user would: ## Selectors -Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it — narrow a large screen with `identifierPrefix`, `subtreeOf` (a ref or an identifier) or `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. +Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. Narrow the node set with `identifierPrefix` or `subtreeOf` (a ref or an identifier), and cut every node down to the keys you read with `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. -Snapshot node fields to filter on: +Snapshot node keys to scan: | Want to find | Scan field | Example value | |---|---|---| +| By semantics identifier | `identifier` | `"nav.tasks"` | | By visible label / text | `label` | `"Login"` | | By value or hint | `value` / `hint` | `"user@example.com"` | -| By tooltip | `tooltip` | `"Close"` | -| By widget key | `key` | `"[<'submitBtn'>]"` | -| By semantic role / type | `flags` or `actions` | `["tap"]` | +| By semantic role / type | `actions` | `["tap"]` | Example — find the "Login" button ref: ``` diff --git a/mcp_server_dart/test/command_catalog_test.dart b/mcp_server_dart/test/command_catalog_test.dart index ff375127..2ceacd62 100644 --- a/mcp_server_dart/test/command_catalog_test.dart +++ b/mcp_server_dart/test/command_catalog_test.dart @@ -223,7 +223,7 @@ void main() { test('rejects semantic_snapshot fields that are not an array', () { expect( () => catalog.buildCommand('semantic_snapshot', {'fields': 'label'}), - throwsA(anything), + throwsA(isA()), ); expect( () => catalog.buildCommand('semantic_snapshot', { diff --git a/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart b/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart index 60c824ff..a55284e2 100644 --- a/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart +++ b/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart @@ -7,6 +7,7 @@ import 'package:flutter/widgets.dart'; import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart' show semanticSnapshotNodeFields; +import 'package:from_json_to_json/from_json_to_json.dart'; import 'background_frame_pump.dart'; @@ -15,31 +16,65 @@ import 'background_frame_pump.dart'; /// The tree is always walked whole — refs number the full walk, so a ref /// read off a filtered snapshot stays valid for every interaction tool. The /// filter only decides which nodes, and which of their fields, are returned. -class SemanticSnapshotFilter { - const SemanticSnapshotFilter({ - this.identifierPrefix, - this.subtreeOf, - this.fields, +/// +/// The wrapped map is what the snapshot echoes back as `filter`: a key is +/// there only when the caller asked for it. +extension type const SemanticSnapshotFilter._(Map value) { + factory SemanticSnapshotFilter({ + final String? identifierPrefix, + final String? subtreeOf, + final List? fields, + }) => SemanticSnapshotFilter._({ + 'identifierPrefix': ?identifierPrefix, + 'subtreeOf': ?subtreeOf, + 'fields': ?fields, }); + /// Reads the filter out of the arguments of a tool call. + /// + /// A legacy handler receives every argument as a string — the toolkit + /// wrapper re-encodes a list as JSON — so `fields` is read from text as + /// readily as from a list. A selector is taken as the caller wrote it, + /// spaces and all: identifiers are matched whole and prefixes with + /// `startsWith`, so trimming would select something else. Only a blank one + /// counts as unasked. + factory SemanticSnapshotFilter.fromJson(final Map json) { + final identifierPrefix = jsonDecodeString(json['identifierPrefix']); + final subtreeOf = jsonDecodeString(json['subtreeOf']); + final fields = jsonDecodeListAs(json['fields']); + return SemanticSnapshotFilter( + identifierPrefix: identifierPrefix.trim().isEmpty + ? null + : identifierPrefix, + subtreeOf: subtreeOf.trim().isEmpty ? null : subtreeOf, + fields: fields.isEmpty ? null : fields, + ); + } + /// Keep nodes whose identifier starts with this prefix. - final String? identifierPrefix; + String? get identifierPrefix => _selector(value['identifierPrefix']); /// Keep one node and its descendants: a ref from the latest snapshot or a /// Semantics identifier, the ref tried first. - final String? subtreeOf; + String? get subtreeOf => _selector(value['subtreeOf']); - /// Node keys to return; `ref` is always kept. - final List? fields; + /// Node keys to return; `ref` is always kept. Absent means every key. + List? get fields { + final decoded = jsonDecodeListAs(value['fields']); + return decoded.isEmpty ? null : decoded; + } bool get isEmpty => identifierPrefix == null && subtreeOf == null && fields == null; - Map toMap() => { - if (identifierPrefix != null) 'identifierPrefix': identifierPrefix, - if (subtreeOf != null) 'subtreeOf': subtreeOf, - if (fields != null) 'fields': fields, - }; + Map toJson() => value; + + static const empty = SemanticSnapshotFilter._({}); + + static String? _selector(final Object? raw) { + final text = jsonDecodeString(raw); + return text.trim().isEmpty ? null : text; + } } /// A service that walks the Flutter semantics tree and produces a compact, @@ -365,9 +400,18 @@ mixin SemanticSnapshotService { final subtreeOf = filter?.subtreeOf; SemanticsNode? subtreeRoot; if (subtreeOf != null) { + final currentRoot = _currentRootNode(); + final cached = _lastRefMap[subtreeOf]; + // A ref outlives the node it names: the widget can be gone while the + // map still points at the detached SemanticsNode. Filtering by that + // node keeps nothing, and an empty snapshot reads as a screen that + // went blank instead of the expired ref it is. subtreeRoot = - _lastRefMap[subtreeOf] ?? - _findByIdentifier(_currentRootNode(), subtreeOf); + cached != null && + currentRoot != null && + _isWithin(cached, currentRoot) + ? cached + : _findByIdentifier(currentRoot, subtreeOf); if (subtreeRoot == null) { return { 'success': false, @@ -545,7 +589,7 @@ mixin SemanticSnapshotService { 'nodeCount': returned.length, if (filter != null && !filter.isEmpty) ...{ 'totalNodeCount': totalNodeCount, - 'filter': filter.toMap(), + 'filter': filter.toJson(), }, 'truncated': truncated, // The surface describes the app, not the slice asked for. diff --git a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart index 3dd258f9..e0d11108 100644 --- a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart +++ b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart @@ -1,5 +1,3 @@ -import 'dart:convert'; - import 'package:dart_mcp/client.dart'; import 'package:flutter/gestures.dart' show PointerDeviceKind; import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart'; @@ -94,11 +92,7 @@ extension type OnSemanticSnapshotEntry._(AgentCallEntry entry) factory OnSemanticSnapshotEntry() { final entry = mcpToolkitTool( handler: (final parameters) async { - final filter = SemanticSnapshotFilter( - identifierPrefix: _nonEmptyString(parameters['identifierPrefix']), - subtreeOf: _nonEmptyString(parameters['subtreeOf']), - fields: _stringList(parameters['fields']), - ); + final filter = SemanticSnapshotFilter.fromJson(parameters); final snapshot = await SemanticSnapshotService.buildSemanticSnapshot( filter: filter.isEmpty ? null : filter, ); @@ -883,22 +877,3 @@ extension type OnFocusWidgetEntry._(AgentCallEntry entry) return OnFocusWidgetEntry._(entry); } } - -/// A list argument reaches a legacy handler JSON-encoded — the wire map holds -/// strings — and already validated against the schema, so the text is JSON. -List? _stringList(final Object? value) { - final decoded = switch (value) { - final String text when text.trim().isNotEmpty => jsonDecode(text), - final List list => list, - _ => null, - }; - return decoded is List - ? decoded.map((final f) => f.toString()).toList() - : null; -} - -String? _nonEmptyString(final Object? value) { - if (value is! String) return null; - final trimmed = value.trim(); - return trimmed.isEmpty ? null : trimmed; -} diff --git a/mcp_toolkit/test/semantic_snapshot_surface_test.dart b/mcp_toolkit/test/semantic_snapshot_surface_test.dart index 3edba952..c3e478e4 100644 --- a/mcp_toolkit/test/semantic_snapshot_surface_test.dart +++ b/mcp_toolkit/test/semantic_snapshot_surface_test.dart @@ -565,7 +565,7 @@ void main() { }; final future = SemanticSnapshotService.buildSemanticSnapshot( - filter: const SemanticSnapshotFilter(identifierPrefix: 'nav.'), + filter: SemanticSnapshotFilter(identifierPrefix: 'nav.'), ); await tester.pump(); final filtered = await future; @@ -604,7 +604,7 @@ void main() { // By identifier: no earlier snapshot is needed. var future = SemanticSnapshotService.buildSemanticSnapshot( - filter: const SemanticSnapshotFilter(subtreeOf: 'panel'), + filter: SemanticSnapshotFilter(subtreeOf: 'panel'), ); await tester.pump(); final byIdentifier = await future; @@ -657,7 +657,7 @@ void main() { final knownRef = nodesOf(before).first['ref']! as String; final future = SemanticSnapshotService.buildSemanticSnapshot( - filter: const SemanticSnapshotFilter(subtreeOf: 'no.such.node'), + filter: SemanticSnapshotFilter(subtreeOf: 'no.such.node'), ); await tester.pump(); final result = await future; @@ -677,6 +677,45 @@ void main() { }, ); + testWidgets('a ref whose node is gone is refused, not answered empty', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final before = await _snapshotAfterPump(tester); + final panelRef = + nodesOf( + before, + ).firstWhere((final n) => n['identifier'] == 'panel')['ref']! + as String; + + // The panel leaves the tree; the ref map still points at its node. + await tester.pumpWidget( + const MaterialApp( + home: Scaffold(body: Center(child: Text('Nothing here'))), + ), + ); + await tester.pumpAndSettle(); + + final future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(subtreeOf: panelRef), + ); + await tester.pump(); + final result = await future; + + expect(result['success'], isFalse); + expect(result['error'], 'subtree_root_not_found'); + expect( + SemanticSnapshotService.currentSnapshotId, + before['snapshot_id'], + reason: 'a detached root spends no snapshot either', + ); + } finally { + semantics.dispose(); + } + }); + testWidgets('the entry decodes fields from the wire map', ( final tester, ) async { @@ -716,7 +755,7 @@ void main() { await pumpRailAndPanel(tester); var future = SemanticSnapshotService.buildSemanticSnapshot( - filter: const SemanticSnapshotFilter( + filter: SemanticSnapshotFilter( fields: ['identifier', 'label'], ), ); @@ -730,7 +769,7 @@ void main() { } future = SemanticSnapshotService.buildSemanticSnapshot( - filter: const SemanticSnapshotFilter(fields: ['ref', 'nope']), + filter: SemanticSnapshotFilter(fields: ['ref', 'nope']), ); await tester.pump(); final refused = await future; diff --git a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md index 23cf2b36..563d6b80 100644 --- a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md @@ -19,17 +19,16 @@ Use this skill when you need to drive a running Flutter app as a user would: ## Selectors -Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it — narrow a large screen with `identifierPrefix`, `subtreeOf` (a ref or an identifier) or `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. +Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. Narrow the node set with `identifierPrefix` or `subtreeOf` (a ref or an identifier), and cut every node down to the keys you read with `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. -Snapshot node fields to filter on: +Snapshot node keys to scan: | Want to find | Scan field | Example value | |---|---|---| +| By semantics identifier | `identifier` | `"nav.tasks"` | | By visible label / text | `label` | `"Login"` | | By value or hint | `value` / `hint` | `"user@example.com"` | -| By tooltip | `tooltip` | `"Close"` | -| By widget key | `key` | `"[<'submitBtn'>]"` | -| By semantic role / type | `flags` or `actions` | `["tap"]` | +| By semantic role / type | `actions` | `["tap"]` | Example — find the "Login" button ref: ``` From 6289039f1009765f1b1940d66d9158e4ff281021 Mon Sep 17 00:00:00 2001 From: Dmitry Dolgopyatov Date: Tue, 8 Sep 2026 11:17:28 +0300 Subject: [PATCH 6/6] docs(control): tell a node's role from what it accepts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scan table paired "semantic role / type" with `actions`, which lists the interactions a node accepts, not what it is. `type` is the key that names the role — button, textField, slider — so each gets its own row. --- mcp_server_dart/lib/src/skill_assets.g.dart | 3 ++- plugin/skills/flutter-mcp-toolkit-control/SKILL.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mcp_server_dart/lib/src/skill_assets.g.dart b/mcp_server_dart/lib/src/skill_assets.g.dart index bef5a600..21a8e138 100644 --- a/mcp_server_dart/lib/src/skill_assets.g.dart +++ b/mcp_server_dart/lib/src/skill_assets.g.dart @@ -606,7 +606,8 @@ Snapshot node keys to scan: | By semantics identifier | `identifier` | `"nav.tasks"` | | By visible label / text | `label` | `"Login"` | | By value or hint | `value` / `hint` | `"user@example.com"` | -| By semantic role / type | `actions` | `["tap"]` | +| By semantic role | `type` | `"button"` | +| By what it accepts | `actions` | `["tap"]` | Example — find the "Login" button ref: ``` diff --git a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md index 563d6b80..26835c0c 100644 --- a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md @@ -28,7 +28,8 @@ Snapshot node keys to scan: | By semantics identifier | `identifier` | `"nav.tasks"` | | By visible label / text | `label` | `"Login"` | | By value or hint | `value` / `hint` | `"user@example.com"` | -| By semantic role / type | `actions` | `["tap"]` | +| By semantic role | `type` | `"button"` | +| By what it accepts | `actions` | `["tap"]` | Example — find the "Login" button ref: ```