diff --git a/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart b/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart index 195389ba..a48efe3e 100644 --- a/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart +++ b/mcp_server_dart/lib/src/capabilities/dynamic_registry/dynamic_gateway.dart @@ -427,6 +427,15 @@ CoreResult? validationFailureForDynamicSchema({ code: CoreErrorCode.invalidCommand, message: e.message, ); + } on FormatException catch (e) { + // An array or object argument travels as wire text and is decoded while + // coercing. Text that is not JSON throws out of the decoder itself, and + // one mistyped argument must read back as an invalid command, not as a + // crash from inside validation. + return CoreResult.failure( + code: CoreErrorCode.invalidCommand, + message: 'Invalid argument for $subjectLabel: ${e.message}', + ); } } diff --git a/mcp_server_dart/lib/src/shared_core/command_executor.dart b/mcp_server_dart/lib/src/shared_core/command_executor.dart index 1bf8d34f..9a522bd0 100644 --- a/mcp_server_dart/lib/src/shared_core/command_executor.dart +++ b/mcp_server_dart/lib/src/shared_core/command_executor.dart @@ -331,7 +331,7 @@ final class DefaultCoreCommandExecutor implements CoreCommandExecutor { GetViewDetailsCommand() => _getViewDetails(), InspectWidgetAtPointCommand() => _inspectWidgetAtPoint(command), CaptureUiSnapshotCommand() => _captureUiSnapshot(command), - SemanticSnapshotCommand() => _semanticSnapshot(), + SemanticSnapshotCommand() => _semanticSnapshot(command), TapWidgetCommand() => _tapWidget(command), EnterTextCommand() => _enterText(command), RevealSearchCommand() => _revealSearch(command), @@ -930,15 +930,25 @@ final class DefaultCoreCommandExecutor implements CoreCommandExecutor { } } - Future _semanticSnapshot() async { + Future _semanticSnapshot( + final SemanticSnapshotCommand command, + ) async { final ensureFailure = await _ensureVmConnected(); if (ensureFailure != null) return ensureFailure; try { final result = await connectionContext.callFlutterExtension( mcpToolkitExtKeys.semanticSnapshot, + args: { + if (command.identifierPrefix != null) + 'identifierPrefix': command.identifierPrefix, + if (command.subtreeOf != null) 'subtreeOf': command.subtreeOf, + // Extension parameters travel as strings; the app decodes a + // list from JSON, not from List.toString(). + if (command.fields != null) 'fields': jsonEncode(command.fields), + }, ); - return CoreResult.success(data: _map(result.json)); + return routeSemanticSnapshotResponse(_map(result.json)); } on Exception catch (e) { return CoreResult.failure( code: CoreErrorCode.semanticSnapshotFailed, @@ -2027,6 +2037,22 @@ CoreResult routeInteractionResponse( ); } +/// Route a `semantic_snapshot` payload to a result that matches its verdict. +/// +/// A captured snapshot carries no `success` key — only a refusal does, so an +/// absent key is a success. A filter that names an unresolvable `subtreeOf` +/// takes no snapshot at all, and reaching the caller as a successful command +/// would let it read stale refs as if they had just been issued. +CoreResult routeSemanticSnapshotResponse(final Map data) { + if (data['success'] != false) return CoreResult.success(data: data); + return CoreResult.failure( + code: CoreErrorCode.semanticSnapshotFailed, + message: + 'semantic_snapshot refused: ${data['error'] ?? 'no reason given'}', + details: data, + ); +} + /// Route the toolkit's `wait_for` extension response to a [CoreResult]. /// /// [data] is the decoded toolkit response. Returns success for diff --git a/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart b/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart index 380ddbe1..637693e2 100644 --- a/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart +++ b/mcp_server_dart/lib/src/shared_core/commands/commands_catalog.dart @@ -674,7 +674,30 @@ final class CommandCatalog { requiresVm: true, supportsWatch: true, mcpExposed: true, - build: (final args) => const SemanticSnapshotCommand(), + build: (final args) { + // Schema validation has already rejected anything but an array; + // the enum inside it is not enforced there, so name a bad field + // here rather than after a VM round trip. + final rawFields = _findArg(args, 'fields'); + final fields = rawFields is List + ? rawFields.map((final f) => f.toString()).toList() + : null; + final unknown = fields + ?.where((final f) => !semanticSnapshotNodeFields.contains(f)) + .toList(); + if (unknown != null && unknown.isNotEmpty) { + throw ArgumentError( + 'Invalid value for "fields": ${unknown.join(', ')} ' + '(accepted: ${semanticSnapshotNodeFields.join(', ')}; ' + r'schema path: $.inputSchema.properties.fields.items)', + ); + } + return SemanticSnapshotCommand( + identifierPrefix: _nullableStringArg(args, 'identifierPrefix'), + subtreeOf: _nullableStringArg(args, 'subtreeOf'), + fields: fields, + ); + }, ), CommandSpec( name: 'tap_widget', diff --git a/mcp_server_dart/lib/src/skill_assets.g.dart b/mcp_server_dart/lib/src/skill_assets.g.dart index 571b778b..21a8e138 100644 --- a/mcp_server_dart/lib/src/skill_assets.g.dart +++ b/mcp_server_dart/lib/src/skill_assets.g.dart @@ -498,16 +498,25 @@ Returns: `{"extensionRPCs": ["ext.flutter.inspector.getRootWidget", "ext.mcp.too ### semantic_snapshot -Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. +Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. A node is listed when it reads (label, value), acts (button, text field, tap, scroll…) or carries a `Semantics(identifier:)`. +- `identifierPrefix` (string, optional) — keep only nodes whose identifier starts with it (`"nav."` for one rail). +- `subtreeOf` (string, optional) — keep one node and its descendants; a ref from the latest snapshot or an identifier, the ref tried first. +- `fields` (array of strings, optional) — node keys to return; `ref` is always kept. Names: `ref id type identifier label value hint enabled focused checked toggled selected bounds actions children visibleInViewport centerInViewport center`. - `connection` (object, optional) — connection override. +The tree is always walked whole, so a ref read off a filtered snapshot is the same ref the full snapshot would give and works with every interaction tool. A filtered reply adds `totalNodeCount` and echoes `filter`; `children` lists kept refs only. + ``` semantic_snapshot() +semantic_snapshot(identifierPrefix: "nav.", fields: ["identifier", "selected"]) +semantic_snapshot(subtreeOf: "panel.tabs") ``` -Returns: `{"snapshot_id": 3, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"]}]}` +Returns: `{"snapshot_id": 3, "nodeCount": 1, "viewport": {...}, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"], "bounds": {...}, "visibleInViewport": true, "centerInViewport": true, "center": {...}}]}` — the viewport is stated once in the envelope, not on each node. +- `subtree_root_not_found` — `subtreeOf` is neither a ref of the latest snapshot nor an identifier in the tree; no snapshot was taken, refs and `snapshot_id` are unchanged. +- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. Over MCP and the CLI the name is refused at the boundary instead — an invalid `fields` argument naming the accepted keys, without spending a call on the app. - `vm_service_unavailable` — app not running or `MCPToolkitBinding.initialize()` not called. - `connection_selection_required` — multiple targets; supply `connection.targetId`. @@ -588,17 +597,17 @@ Use this skill when you need to drive a running Flutter app as a user would: ## Selectors -Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. +Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. Narrow the node set with `identifierPrefix` or `subtreeOf` (a ref or an identifier), and cut every node down to the keys you read with `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. -Snapshot node fields to filter on: +Snapshot node keys to scan: | Want to find | Scan field | Example value | |---|---|---| +| By semantics identifier | `identifier` | `"nav.tasks"` | | By visible label / text | `label` | `"Login"` | | By value or hint | `value` / `hint` | `"user@example.com"` | -| By tooltip | `tooltip` | `"Close"` | -| By widget key | `key` | `"[<'submitBtn'>]"` | -| By semantic role / type | `flags` or `actions` | `["tap"]` | +| By semantic role | `type` | `"button"` | +| By what it accepts | `actions` | `["tap"]` | Example — find the "Login" button ref: ``` diff --git a/mcp_server_dart/test/command_catalog_test.dart b/mcp_server_dart/test/command_catalog_test.dart index 7c245a29..2ceacd62 100644 --- a/mcp_server_dart/test/command_catalog_test.dart +++ b/mcp_server_dart/test/command_catalog_test.dart @@ -207,6 +207,33 @@ void main() { expect(command, isA()); }); + test('carries semantic_snapshot filters into the command', () { + final command = + catalog.buildCommand('semantic_snapshot', { + 'identifierPrefix': 'nav.', + 'subtreeOf': 's_3', + 'fields': ['identifier', 'label'], + }) + as SemanticSnapshotCommand; + expect(command.identifierPrefix, 'nav.'); + expect(command.subtreeOf, 's_3'); + expect(command.fields, ['identifier', 'label']); + }); + + test('rejects semantic_snapshot fields that are not an array', () { + expect( + () => catalog.buildCommand('semantic_snapshot', {'fields': 'label'}), + throwsA(isA()), + ); + expect( + () => catalog.buildCommand('semantic_snapshot', { + 'fields': ['nope'], + }), + throwsA(isA()), + reason: 'the enum in the schema names the accepted fields', + ); + }); + test('reveal_search command is registered with bounded schema', () { final spec = catalog.specFor('reveal_search'); expect(spec, isNotNull); diff --git a/mcp_server_dart/test/interaction_response_routing_test.dart b/mcp_server_dart/test/interaction_response_routing_test.dart index 18aba698..1e00682a 100644 --- a/mcp_server_dart/test/interaction_response_routing_test.dart +++ b/mcp_server_dart/test/interaction_response_routing_test.dart @@ -83,4 +83,34 @@ void main() { expect(recovery.containsKey('fix_command'), isFalse); }); }); + + group('routeSemanticSnapshotResponse', () { + test('a captured snapshot has no success key and still routes to ok', () { + final result = routeSemanticSnapshotResponse({ + 'snapshot_id': 7, + 'nodes': [ + {'ref': 's_0', 'identifier': 'nav.tasks'}, + ], + }); + + expect(result.ok, isTrue); + expect((result.data! as Map)['snapshot_id'], 7); + }); + + test('an unresolvable subtreeOf is a failure, not an empty snapshot', () { + // No snapshot was taken, so the refs the caller already holds stay + // current — reporting success would read as a screen that went empty. + final result = routeSemanticSnapshotResponse({ + 'success': false, + 'error': 'subtree_root_not_found', + 'subtreeOf': 'panel.missing', + 'hint': 'Take a full snapshot and read the identifier off it.', + }); + + expect(result.ok, isFalse); + expect(result.error!.code, CoreErrorCode.semanticSnapshotFailed); + expect(result.error!.message, contains('subtree_root_not_found')); + expect((result.error!.details! as Map)['subtreeOf'], 'panel.missing'); + }); + }); } diff --git a/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart b/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart index c62a0b26..a55284e2 100644 --- a/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart +++ b/mcp_toolkit/lib/src/services/semantic_snapshot_service.dart @@ -5,8 +5,78 @@ import 'dart:ui' as ui; import 'package:flutter/rendering.dart'; import 'package:flutter/widgets.dart'; +import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart' + show semanticSnapshotNodeFields; +import 'package:from_json_to_json/from_json_to_json.dart'; + import 'background_frame_pump.dart'; +/// What a `semantic_snapshot` call keeps of the full tree. +/// +/// The tree is always walked whole — refs number the full walk, so a ref +/// read off a filtered snapshot stays valid for every interaction tool. The +/// filter only decides which nodes, and which of their fields, are returned. +/// +/// The wrapped map is what the snapshot echoes back as `filter`: a key is +/// there only when the caller asked for it. +extension type const SemanticSnapshotFilter._(Map value) { + factory SemanticSnapshotFilter({ + final String? identifierPrefix, + final String? subtreeOf, + final List? fields, + }) => SemanticSnapshotFilter._({ + 'identifierPrefix': ?identifierPrefix, + 'subtreeOf': ?subtreeOf, + 'fields': ?fields, + }); + + /// Reads the filter out of the arguments of a tool call. + /// + /// A legacy handler receives every argument as a string — the toolkit + /// wrapper re-encodes a list as JSON — so `fields` is read from text as + /// readily as from a list. A selector is taken as the caller wrote it, + /// spaces and all: identifiers are matched whole and prefixes with + /// `startsWith`, so trimming would select something else. Only a blank one + /// counts as unasked. + factory SemanticSnapshotFilter.fromJson(final Map json) { + final identifierPrefix = jsonDecodeString(json['identifierPrefix']); + final subtreeOf = jsonDecodeString(json['subtreeOf']); + final fields = jsonDecodeListAs(json['fields']); + return SemanticSnapshotFilter( + identifierPrefix: identifierPrefix.trim().isEmpty + ? null + : identifierPrefix, + subtreeOf: subtreeOf.trim().isEmpty ? null : subtreeOf, + fields: fields.isEmpty ? null : fields, + ); + } + + /// Keep nodes whose identifier starts with this prefix. + String? get identifierPrefix => _selector(value['identifierPrefix']); + + /// Keep one node and its descendants: a ref from the latest snapshot or a + /// Semantics identifier, the ref tried first. + String? get subtreeOf => _selector(value['subtreeOf']); + + /// Node keys to return; `ref` is always kept. Absent means every key. + List? get fields { + final decoded = jsonDecodeListAs(value['fields']); + return decoded.isEmpty ? null : decoded; + } + + bool get isEmpty => + identifierPrefix == null && subtreeOf == null && fields == null; + + Map toJson() => value; + + static const empty = SemanticSnapshotFilter._({}); + + static String? _selector(final Object? raw) { + final text = jsonDecodeString(raw); + return text.trim().isEmpty ? null : text; + } +} + /// A service that walks the Flutter semantics tree and produces a compact, /// AI-friendly snapshot of interactive / meaningful elements. /// @@ -97,14 +167,20 @@ mixin SemanticSnapshotService { final bounds = resolveBounds(ref); final center = resolveCenter(ref); final viewport = viewportRect; - return visibilityForBounds( - bounds: bounds, - center: center, - viewport: viewport, - ); + return { + ...visibilityForBounds( + bounds: bounds, + center: center, + viewport: viewport, + ), + if (viewport != null) 'viewport': _rectToMap(viewport), + }; } /// Visibility metadata for logical bounds in the current Flutter viewport. + /// + /// The viewport itself is not part of the answer: a snapshot states it once + /// in its envelope, and a single-ref reply adds it in [visibilityForRef]. static Map visibilityForBounds({ required final ui.Rect? bounds, required final ui.Offset? center, @@ -122,7 +198,6 @@ mixin SemanticSnapshotService { 'visibleInViewport': visible, 'centerInViewport': centerVisible, if (bounds != null) 'bounds': _rectToMap(bounds), - if (viewport != null) 'viewport': _rectToMap(viewport), if (center != null) 'center': {'x': center.dx, 'y': center.dy}, }; @@ -172,8 +247,9 @@ mixin SemanticSnapshotService { /// Async so we can await a frame on the (rare) cold path where the /// semantics tree hasn't been primed yet — e.g. if /// [MCPToolkitBinding.initialize] didn't run for some reason. - static Future> buildSemanticSnapshot() => - _buildSnapshot(incrementId: true); + static Future> buildSemanticSnapshot({ + final SemanticSnapshotFilter? filter, + }) => _buildSnapshot(incrementId: true, filter: filter); /// Internal: read the snapshot without bumping the public id stream. /// Used by `WaitPredicateService` while polling so callers' outstanding @@ -245,7 +321,24 @@ mixin SemanticSnapshotService { static Future> _buildSnapshot({ required final bool incrementId, + final SemanticSnapshotFilter? filter, }) async { + final unknownFields = filter?.fields + ?.where((final f) => !semanticSnapshotNodeFields.contains(f)) + .toList(); + if (unknownFields != null && unknownFields.isNotEmpty) { + return { + 'success': false, + 'error': 'unknown_field', + 'unknownFields': unknownFields, + 'acceptedFields': semanticSnapshotNodeFields, + 'hint': + 'fields names keys of a snapshot node, and ' + '${unknownFields.map((final f) => '"$f"').join(', ')} ' + '${unknownFields.length == 1 ? 'is' : 'are'} not among them — ' + 'pick from acceptedFields. No snapshot was taken.', + }; + } final binding = WidgetsBinding.instance; final isTestBinding = _isInFlutterTest(); @@ -279,7 +372,7 @@ mixin SemanticSnapshotService { await pumpFramesIfSuspended(); } - return await _buildSnapshotBody(incrementId: incrementId); + return await _buildSnapshotBody(incrementId: incrementId, filter: filter); } finally { if (isTestBinding) { handle.dispose(); @@ -296,11 +389,45 @@ mixin SemanticSnapshotService { static Future> _buildSnapshotBody({ required final bool incrementId, + final SemanticSnapshotFilter? filter, }) async { final refMap = {}; final boundsMap = {}; final centerMap = {}; + // Resolved before the counter moves and the ref maps are replaced, so a + // refusal leaves the caller's refs and snapshot id exactly as they were. + final subtreeOf = filter?.subtreeOf; + SemanticsNode? subtreeRoot; + if (subtreeOf != null) { + final currentRoot = _currentRootNode(); + final cached = _lastRefMap[subtreeOf]; + // A ref outlives the node it names: the widget can be gone while the + // map still points at the detached SemanticsNode. Filtering by that + // node keeps nothing, and an empty snapshot reads as a screen that + // went blank instead of the expired ref it is. + subtreeRoot = + cached != null && + currentRoot != null && + _isWithin(cached, currentRoot) + ? cached + : _findByIdentifier(currentRoot, subtreeOf); + if (subtreeRoot == null) { + return { + 'success': false, + 'error': 'subtree_root_not_found', + 'subtreeOf': subtreeOf, + 'snapshotId': _snapshotCounter, + 'hint': + '"$subtreeOf" is neither a ref of snapshot $_snapshotCounter ' + 'nor an identifier the tree publishes. Refs expire with every ' + 'snapshot, so pass an identifier when one exists, or take an ' + 'unfiltered snapshot and use a ref from it. No snapshot was ' + 'taken.', + }; + } + } + final snapshotId = incrementId ? ++_snapshotCounter : _snapshotCounter; SemanticsNode? root; @@ -446,16 +573,112 @@ mixin SemanticSnapshotService { ); } + final totalNodeCount = nodes.length; + final returned = filter == null || filter.isEmpty + ? nodes + : _applyFilter( + nodes: nodes, + refMap: refMap, + filter: filter, + subtreeRoot: subtreeRoot, + ); + return { 'snapshot_id': snapshotId, - 'nodes': nodes, - 'nodeCount': nodes.length, + 'nodes': returned, + 'nodeCount': returned.length, + if (filter != null && !filter.isEmpty) ...{ + 'totalNodeCount': totalNodeCount, + 'filter': filter.toJson(), + }, 'truncated': truncated, - 'interactionSurface': _classifyInteractionSurface(nodes.length), + // The surface describes the app, not the slice asked for. + 'interactionSurface': _classifyInteractionSurface(totalNodeCount), if (viewport != null) 'viewport': _rectToMap(viewport), }; } + /// The nodes a [filter] keeps, with `children` pruned to kept refs and + /// fields projected to the requested set. + static List> _applyFilter({ + required final List> nodes, + required final Map refMap, + required final SemanticSnapshotFilter filter, + required final SemanticsNode? subtreeRoot, + }) { + final prefix = filter.identifierPrefix; + final kept = >[ + for (final node in nodes) + if ((subtreeRoot == null || + _isWithin(refMap[node['ref']! as String], subtreeRoot)) && + (prefix == null || + (node['identifier'] is String && + (node['identifier']! as String).startsWith(prefix)))) + node, + ]; + final keptRefs = {for (final node in kept) node['ref']}; + final fields = filter.fields?.toSet(); + return >[ + for (final node in kept) + { + for (final MapEntry(:key, :value) in node.entries) + if (fields == null || key == 'ref' || fields.contains(key)) + if (key == 'children') + key: [ + for (final child in value! as List) + if (keptRefs.contains(child)) child, + ] + else + key: value, + }..removeWhere( + (final key, final value) => + key == 'children' && (value! as List).isEmpty, + ), + ]; + } + + /// Whether [node] is [root] or sits below it. + static bool _isWithin(final SemanticsNode? node, final SemanticsNode root) { + for (var current = node; current != null; current = current.parent) { + if (identical(current, root)) return true; + } + return false; + } + + static SemanticsNode? _currentRootNode() { + try { + final renderViews = WidgetsBinding.instance.renderViews; + if (renderViews.isEmpty) return null; + final owner = (_activeRenderView ?? renderViews.first).owner; + return owner?.semanticsOwner?.rootSemanticsNode; + } on Exception { + // The body reports the missing tree itself; the lookup just found nothing. + return null; + } + } + + static SemanticsNode? _findByIdentifier( + final SemanticsNode? root, + final String identifier, + ) { + if (root == null) return null; + SemanticsNode? found; + void walk(final SemanticsNode node) { + if (found != null) return; + if (node.getSemanticsData().identifier == identifier) { + found = node; + return; + } + node.visitChildren((final child) { + walk(child); + return found == null; + }); + } + + walk(root); + return found; + } + /// How agents should interact with this app's visible surface. /// /// - [flutter_widgets]: semantics expose tappable refs (normal Material/Cupertino). @@ -543,6 +766,11 @@ mixin SemanticSnapshotService { // Has a semantic label or value worth surfacing. if (data.label.isNotEmpty || data.value.isNotEmpty) return true; + // An identifier is a handle the app published on purpose — a container + // named "rail" or "panel" exists to be addressed, by subtreeOf or by + // wait_for, even though it neither reads nor acts. + if (data.identifier.isNotEmpty) return true; + // Interactive flags. if (data.hasFlag(SemanticsFlag.isButton)) return true; if (data.hasFlag(SemanticsFlag.isTextField)) return true; diff --git a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart index 07c1e629..e0d11108 100644 --- a/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart +++ b/mcp_toolkit/lib/src/toolkits/interaction_toolkit.dart @@ -92,10 +92,14 @@ extension type OnSemanticSnapshotEntry._(AgentCallEntry entry) factory OnSemanticSnapshotEntry() { final entry = mcpToolkitTool( handler: (final parameters) async { - final snapshot = await SemanticSnapshotService.buildSemanticSnapshot(); + final filter = SemanticSnapshotFilter.fromJson(parameters); + final snapshot = await SemanticSnapshotService.buildSemanticSnapshot( + filter: filter.isEmpty ? null : filter, + ); return MCPCallResult( - message: - 'Semantic snapshot captured. Use refs to interact with widgets.', + message: snapshot['success'] == false + ? 'Semantic snapshot refused: ${snapshot['error']}' + : 'Semantic snapshot captured. Use refs to interact with widgets.', parameters: snapshot, ); }, @@ -104,6 +108,8 @@ extension type OnSemanticSnapshotEntry._(AgentCallEntry entry) description: 'Get compact semantic tree of interactive widgets with refs ' 'for interaction tools (tap_widget, enter_text, etc.). ' + 'Narrow it with identifierPrefix, subtreeOf (a ref or an ' + 'identifier) and fields; refs are those of the full tree. ' 'A control that appears only under the pointer is absent here: ' 'on desktop and web, hover the element that should own it and ' 'snapshot again.', diff --git a/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart b/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart index 7a1d39a5..ae41a97f 100644 --- a/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart +++ b/mcp_toolkit/test/interaction_toolkit_schema_parity_test.dart @@ -55,6 +55,12 @@ void main() { expect(schema.containsKey('required'), isFalse); final props = schema['properties']! as Map; expect(props.containsKey('connection'), isTrue); + expect(props.containsKey('identifierPrefix'), isTrue); + expect(props.containsKey('subtreeOf'), isTrue); + final fields = props['fields']! as Map; + final items = fields['items']! as Map; + expect(items['enum'], semanticSnapshotNodeFields); + expect(semanticSnapshotNodeFields, contains('ref')); }); test('wait_for', () { diff --git a/mcp_toolkit/test/semantic_snapshot_surface_test.dart b/mcp_toolkit/test/semantic_snapshot_surface_test.dart index 4dcaddca..c3e478e4 100644 --- a/mcp_toolkit/test/semantic_snapshot_surface_test.dart +++ b/mcp_toolkit/test/semantic_snapshot_surface_test.dart @@ -1,4 +1,6 @@ import 'package:flutter/material.dart'; +import 'package:flutter_mcp_toolkit_core/flutter_mcp_toolkit_core.dart' + show semanticSnapshotNodeFields; import 'package:flutter_test/flutter_test.dart'; import 'package:mcp_toolkit/mcp_toolkit.dart'; @@ -461,6 +463,326 @@ void main() { }, ); + group('semantic_snapshot filters', () { + Future pumpRailAndPanel(final WidgetTester tester) async { + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: Row( + children: [ + Semantics( + identifier: 'rail', + container: true, + child: Column( + children: [ + Semantics( + identifier: 'nav.tasks', + button: true, + onTap: () {}, + child: const Text('Tasks'), + ), + Semantics( + identifier: 'nav.calendar', + button: true, + onTap: () {}, + child: const Text('Calendar'), + ), + ], + ), + ), + Semantics( + identifier: 'panel', + container: true, + child: Column( + children: [ + Semantics( + identifier: 'panel.tab.overview', + button: true, + onTap: () {}, + child: const Text('Overview'), + ), + // No identifier and no boundary of its own: the label + // still has to travel with the button node. + Semantics( + container: true, + button: true, + onTap: () {}, + child: const Text('Unnamed button'), + ), + ], + ), + ), + ], + ), + ), + ), + ); + await tester.pumpAndSettle(); + } + + List> nodesOf(final Map snapshot) => + (snapshot['nodes']! as List).cast>(); + + testWidgets('the viewport is stated once, not on every node', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final snapshot = await _snapshotAfterPump(tester); + + expect(snapshot['viewport'], isA>()); + for (final node in nodesOf(snapshot)) { + expect( + node.containsKey('viewport'), + isFalse, + reason: '${node['ref']}', + ); + expect(node.containsKey('visibleInViewport'), isTrue); + } + // A single-ref answer still carries it, since it has no envelope. + final ref = nodesOf(snapshot).first['ref']! as String; + expect( + SemanticSnapshotService.visibilityForRef(ref)['viewport'], + isA>(), + ); + } finally { + semantics.dispose(); + } + }); + + testWidgets('identifierPrefix keeps the matching nodes only', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final full = await _snapshotAfterPump(tester); + final fullRefs = { + for (final node in nodesOf(full)) + if (node['identifier'] is String) + node['identifier']! as String: node['ref']! as String, + }; + + final future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(identifierPrefix: 'nav.'), + ); + await tester.pump(); + final filtered = await future; + + final nodes = nodesOf(filtered); + expect( + nodes.map((final n) => n['identifier']), + unorderedEquals(['nav.tasks', 'nav.calendar']), + ); + // Refs number the full walk, so they agree with the unfiltered + // snapshot and every interaction tool can resolve them. + for (final node in nodes) { + expect(node['ref'], fullRefs[node['identifier']]); + expect( + SemanticSnapshotService.resolveRef(node['ref']! as String), + isNotNull, + ); + } + expect(filtered['nodeCount'], 2); + expect(filtered['totalNodeCount'], nodesOf(full).length); + expect(filtered['filter'], { + 'identifierPrefix': 'nav.', + }); + expect(filtered['interactionSurface'], full['interactionSurface']); + } finally { + semantics.dispose(); + } + }); + + testWidgets('subtreeOf narrows to a node and its descendants', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + + // By identifier: no earlier snapshot is needed. + var future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(subtreeOf: 'panel'), + ); + await tester.pump(); + final byIdentifier = await future; + final panelNodes = nodesOf(byIdentifier); + expect( + panelNodes.map((final n) => n['identifier'] ?? n['label']), + unorderedEquals([ + 'panel', + 'panel.tab.overview', + 'Unnamed button', + ]), + ); + // Children of the kept container name kept refs only. + final panel = panelNodes.firstWhere( + (final n) => n['identifier'] == 'panel', + ); + final keptRefs = panelNodes.map((final n) => n['ref']).toSet(); + expect( + keptRefs.containsAll(panel['children']! as List), + isTrue, + ); + + // By ref from the latest snapshot. + final railRef = + nodesOf( + await _snapshotAfterPump(tester), + ).firstWhere((final n) => n['identifier'] == 'rail')['ref']! + as String; + future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(subtreeOf: railRef), + ); + await tester.pump(); + final byRef = await future; + expect( + nodesOf(byRef).map((final n) => n['identifier']), + unorderedEquals(['rail', 'nav.tasks', 'nav.calendar']), + ); + } finally { + semantics.dispose(); + } + }); + + testWidgets( + 'an unknown subtree root is refused before a snapshot is spent', + (final tester) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final before = await _snapshotAfterPump(tester); + final knownRef = nodesOf(before).first['ref']! as String; + + final future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(subtreeOf: 'no.such.node'), + ); + await tester.pump(); + final result = await future; + + expect(result['success'], isFalse); + expect(result['error'], 'subtree_root_not_found'); + expect(result['hint'], contains('No snapshot was taken')); + expect( + SemanticSnapshotService.currentSnapshotId, + before['snapshot_id'], + reason: 'a refusal must not move the counter', + ); + expect(SemanticSnapshotService.resolveRef(knownRef), isNotNull); + } finally { + semantics.dispose(); + } + }, + ); + + testWidgets('a ref whose node is gone is refused, not answered empty', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + final before = await _snapshotAfterPump(tester); + final panelRef = + nodesOf( + before, + ).firstWhere((final n) => n['identifier'] == 'panel')['ref']! + as String; + + // The panel leaves the tree; the ref map still points at its node. + await tester.pumpWidget( + const MaterialApp( + home: Scaffold(body: Center(child: Text('Nothing here'))), + ), + ); + await tester.pumpAndSettle(); + + final future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(subtreeOf: panelRef), + ); + await tester.pump(); + final result = await future; + + expect(result['success'], isFalse); + expect(result['error'], 'subtree_root_not_found'); + expect( + SemanticSnapshotService.currentSnapshotId, + before['snapshot_id'], + reason: 'a detached root spends no snapshot either', + ); + } finally { + semantics.dispose(); + } + }); + + testWidgets('the entry decodes fields from the wire map', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + // The service extension hands a legacy handler strings only, so the + // list arrives JSON-encoded; invokeDirect walks that same path. + final future = OnSemanticSnapshotEntry().invokeDirect({ + 'identifierPrefix': 'nav.', + 'fields': ['identifier', 'label'], + }); + await tester.pump(); + final result = await future; + + expect(result.ok, isTrue, reason: result.message); + expect(result.data['filter'], { + 'identifierPrefix': 'nav.', + 'fields': ['identifier', 'label'], + }); + for (final node in nodesOf(result.data)) { + expect( + node.keys, + unorderedEquals(['ref', 'identifier', 'label']), + ); + } + } finally { + semantics.dispose(); + } + }); + + testWidgets('fields projects every node and always keeps the ref', ( + final tester, + ) async { + final semantics = tester.ensureSemantics(); + try { + await pumpRailAndPanel(tester); + + var future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter( + fields: ['identifier', 'label'], + ), + ); + await tester.pump(); + final projected = await future; + for (final node in nodesOf(projected)) { + expect(node.keys, isNot(contains('bounds'))); + expect(node.keys, isNot(contains('actions'))); + expect(node.keys, contains('ref')); + expect(node.keys, anyOf(contains('identifier'), contains('label'))); + } + + future = SemanticSnapshotService.buildSemanticSnapshot( + filter: SemanticSnapshotFilter(fields: ['ref', 'nope']), + ); + await tester.pump(); + final refused = await future; + expect(refused['success'], isFalse); + expect(refused['error'], 'unknown_field'); + expect(refused['unknownFields'], ['nope']); + expect(refused['acceptedFields'], semanticSnapshotNodeFields); + expect(refused['hint'], contains('"nope"')); + } finally { + semantics.dispose(); + } + }); + }); testWidgets( 'semantic_snapshot reports hybrid when no interactive semantics refs', (final tester) async { diff --git a/packages/core/lib/src/commands/core_commands.dart b/packages/core/lib/src/commands/core_commands.dart index ebf5ac45..a8997079 100644 --- a/packages/core/lib/src/commands/core_commands.dart +++ b/packages/core/lib/src/commands/core_commands.dart @@ -295,7 +295,15 @@ final class CaptureUiSnapshotCommand extends CoreCommand { } final class SemanticSnapshotCommand extends CoreCommand { - const SemanticSnapshotCommand(); + const SemanticSnapshotCommand({ + this.identifierPrefix, + this.subtreeOf, + this.fields, + }); + + final String? identifierPrefix; + final String? subtreeOf; + final List? fields; @override String get name => 'semantic_snapshot'; diff --git a/packages/core/lib/src/tools/interaction_input_schemas.dart b/packages/core/lib/src/tools/interaction_input_schemas.dart index d664bea2..24147ae4 100644 --- a/packages/core/lib/src/tools/interaction_input_schemas.dart +++ b/packages/core/lib/src/tools/interaction_input_schemas.dart @@ -44,11 +44,64 @@ Map tapWidgetInputSchema() => { }, }; +/// Every key a `semantic_snapshot` node can carry, in the order the node +/// publishes them. `fields` selects among these; `ref` is always returned. +const List semanticSnapshotNodeFields = [ + 'ref', + 'id', + 'type', + 'identifier', + 'label', + 'value', + 'hint', + 'enabled', + 'focused', + 'checked', + 'toggled', + 'selected', + 'bounds', + 'actions', + 'children', + 'visibleInViewport', + 'centerInViewport', + 'center', +]; + /// Shared JSON Schema for [semantic_snapshot] / `fmt_semantic_snapshot`. Map semanticSnapshotInputSchema() => { 'type': 'object', 'additionalProperties': false, - 'properties': {'connection': connectionOverrideJsonSchema()}, + 'properties': { + 'identifierPrefix': { + 'type': 'string', + 'description': + 'Return only nodes whose Semantics identifier starts with this ' + 'prefix ("nav." for one navigation rail). Refs are those of the ' + 'full tree, so a ref from a filtered snapshot works everywhere.', + }, + 'subtreeOf': { + 'type': 'string', + 'description': + 'Return only this node and its descendants. A ref from the latest ' + 'snapshot ("s_12") or a Semantics identifier; a ref is tried ' + 'first. Fails with subtree_root_not_found without spending a ' + 'snapshot when neither resolves.', + }, + 'fields': { + 'type': 'array', + 'items': { + 'type': 'string', + 'enum': semanticSnapshotNodeFields, + }, + 'minItems': 1, + 'uniqueItems': true, + 'description': + 'Node fields to return; "ref" is always included. Omit for every ' + 'field. A name outside the list is refused before the call reaches ' + 'the app.', + }, + 'connection': connectionOverrideJsonSchema(), + }, }; /// Shared JSON Schema for [wait_for] / `fmt_wait_for`. diff --git a/packages/server_capability_core/lib/src/tools/semantic_tools.dart b/packages/server_capability_core/lib/src/tools/semantic_tools.dart index 26402cfb..017bdc3c 100644 --- a/packages/server_capability_core/lib/src/tools/semantic_tools.dart +++ b/packages/server_capability_core/lib/src/tools/semantic_tools.dart @@ -23,7 +23,18 @@ void registerSemanticTools(final CapabilityContext context) { 'snapshot again.', inputSchema: semanticSnapshotInputSchema(), handler: (final args) async { - return runCommand(runner, args, const SemanticSnapshotCommand()); + final rawFields = args['fields']; + return runCommand( + runner, + args, + SemanticSnapshotCommand( + identifierPrefix: stringArgOrNull(args['identifierPrefix']), + subtreeOf: stringArgOrNull(args['subtreeOf']), + fields: rawFields is List + ? rawFields.map((final f) => f.toString()).toList() + : null, + ), + ); }, ), ); diff --git a/packages/server_capability_core/test/tools/semantic_tools_test.dart b/packages/server_capability_core/test/tools/semantic_tools_test.dart index de1e0169..2dd68769 100644 --- a/packages/server_capability_core/test/tools/semantic_tools_test.dart +++ b/packages/server_capability_core/test/tools/semantic_tools_test.dart @@ -86,6 +86,37 @@ void main() { }, ); + test( + 'semantic_snapshot handler carries the filters into the command', + () async { + final runner = FakeCommandRunner() + ..nextExecuteResult = CoreResult.success(data: {}); + final ctx = _registeredCtx(runner: runner); + final reg = ctx.registrationFor('semantic_snapshot')!; + await reg.handler(const { + 'identifierPrefix': 'nav.', + 'subtreeOf': 'panel', + 'fields': ['identifier', 'selected'], + }); + final cmd = runner.executedCommands.single as SemanticSnapshotCommand; + expect(cmd.identifierPrefix, 'nav.'); + expect(cmd.subtreeOf, 'panel'); + expect(cmd.fields, ['identifier', 'selected']); + }, + ); + + test('semantic_snapshot handler leaves absent filters null', () async { + final runner = FakeCommandRunner() + ..nextExecuteResult = CoreResult.success(data: {}); + final ctx = _registeredCtx(runner: runner); + final reg = ctx.registrationFor('semantic_snapshot')!; + await reg.handler(const {'identifierPrefix': ' '}); + final cmd = runner.executedCommands.single as SemanticSnapshotCommand; + expect(cmd.identifierPrefix, isNull); + expect(cmd.subtreeOf, isNull); + expect(cmd.fields, isNull); + }); + test( 'semantic_snapshot handler calls applyConnectionOverride before execute', () async { diff --git a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md index 66b6dd15..26835c0c 100644 --- a/plugin/skills/flutter-mcp-toolkit-control/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-control/SKILL.md @@ -19,17 +19,17 @@ Use this skill when you need to drive a running Flutter app as a user would: ## Selectors -Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. +Most interaction tools target a widget by **ref** — a short string like `"s_0"` returned by `semantic_snapshot`. For visible widgets, call `semantic_snapshot`, scan the returned nodes, find the right ref, then pass it. Narrow the node set with `identifierPrefix` or `subtreeOf` (a ref or an identifier), and cut every node down to the keys you read with `fields`; refs are those of the full tree either way. For off-screen targets with stable semantics text or identifier, use `reveal_search`; it performs a bounded snapshot → match → scroll loop and returns a fresh `ref`/`snapshotId`. -Snapshot node fields to filter on: +Snapshot node keys to scan: | Want to find | Scan field | Example value | |---|---|---| +| By semantics identifier | `identifier` | `"nav.tasks"` | | By visible label / text | `label` | `"Login"` | | By value or hint | `value` / `hint` | `"user@example.com"` | -| By tooltip | `tooltip` | `"Close"` | -| By widget key | `key` | `"[<'submitBtn'>]"` | -| By semantic role / type | `flags` or `actions` | `["tap"]` | +| By semantic role | `type` | `"button"` | +| By what it accepts | `actions` | `["tap"]` | Example — find the "Login" button ref: ``` diff --git a/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md b/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md index e199ef2f..6565741a 100644 --- a/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md +++ b/plugin/skills/flutter-mcp-toolkit-inspect/SKILL.md @@ -159,16 +159,25 @@ Returns: `{"extensionRPCs": ["ext.flutter.inspector.getRootWidget", "ext.mcp.too ### semantic_snapshot -Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. +Return a compact accessibility tree of interactive widgets with stable `ref` strings and a `snapshot_id`. A node is listed when it reads (label, value), acts (button, text field, tap, scroll…) or carries a `Semantics(identifier:)`. +- `identifierPrefix` (string, optional) — keep only nodes whose identifier starts with it (`"nav."` for one rail). +- `subtreeOf` (string, optional) — keep one node and its descendants; a ref from the latest snapshot or an identifier, the ref tried first. +- `fields` (array of strings, optional) — node keys to return; `ref` is always kept. Names: `ref id type identifier label value hint enabled focused checked toggled selected bounds actions children visibleInViewport centerInViewport center`. - `connection` (object, optional) — connection override. +The tree is always walked whole, so a ref read off a filtered snapshot is the same ref the full snapshot would give and works with every interaction tool. A filtered reply adds `totalNodeCount` and echoes `filter`; `children` lists kept refs only. + ``` semantic_snapshot() +semantic_snapshot(identifierPrefix: "nav.", fields: ["identifier", "selected"]) +semantic_snapshot(subtreeOf: "panel.tabs") ``` -Returns: `{"snapshot_id": 3, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"]}]}` +Returns: `{"snapshot_id": 3, "nodeCount": 1, "viewport": {...}, "nodes": [{"ref": "s_0", "label": "Increment", "actions": ["tap"], "bounds": {...}, "visibleInViewport": true, "centerInViewport": true, "center": {...}}]}` — the viewport is stated once in the envelope, not on each node. +- `subtree_root_not_found` — `subtreeOf` is neither a ref of the latest snapshot nor an identifier in the tree; no snapshot was taken, refs and `snapshot_id` are unchanged. +- `unknown_field` — a name in `fields` is not a node key; `acceptedFields` lists them. Over MCP and the CLI the name is refused at the boundary instead — an invalid `fields` argument naming the accepted keys, without spending a call on the app. - `vm_service_unavailable` — app not running or `MCPToolkitBinding.initialize()` not called. - `connection_selection_required` — multiple targets; supply `connection.targetId`.