diff --git a/Dockerfile b/Dockerfile index e0766618..03a3fffb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,8 @@ FROM ${SWIPL_IMAGE} AS builder SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV DEBIAN_FRONTEND=noninteractive \ HF_HOME=/opt/huggingface \ - SENTENCE_TRANSFORMERS_HOME=/opt/sentence_transformers + SENTENCE_TRANSFORMERS_HOME=/opt/sentence_transformers \ + PLAYWRIGHT_BROWSERS_PATH=/opt/ms-playwright RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -28,6 +29,16 @@ RUN apt-get update \ nano \ && rm -rf /var/lib/apt/lists/* +# Install pinned Python dependencies and cache the matching Chromium build +# before cloning source repositories or copying application code. +COPY ./requirements.txt /tmp/requirements.txt +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + --index-url https://download.pytorch.org/whl/cpu \ + --extra-index-url https://pypi.org/simple/ \ + torch==2.12.1 \ + && python3 -m pip install --no-cache-dir --break-system-packages -r /tmp/requirements.txt +RUN python3 -m playwright install --only-shell chromium + # Build dependencies from source. Pin refs at build time for reproducibility. ARG PETTA_REPO=https://github.com/trueagi-io/PeTTa.git ARG PETTA_REF=v1.0.4 @@ -52,13 +63,6 @@ RUN sh build.sh RUN mkdir -p /PeTTa/repos \ && git clone --depth 1 --branch "${CHROMADB_REF}" "${CHROMADB_REPO}" /PeTTa/repos/petta_lib_chromadb -COPY ./requirements.txt /tmp/requirements.txt -RUN python3 -m pip install --no-cache-dir --break-system-packages \ - --index-url https://download.pytorch.org/whl/cpu \ - --extra-index-url https://pypi.org/simple/ \ - torch==2.12.1 \ - && python3 -m pip install --no-cache-dir --break-system-packages -r /tmp/requirements.txt - # Pre-download the sentence-transformers model so runtime does not need network access. RUN mkdir -p "${HF_HOME}" "${SENTENCE_TRANSFORMERS_HOME}" \ && python3 - <-*.png` filename. Returns the + full saved path, byte count, and tab ID, or `BROWSER-SCREENSHOT-FAILED` with + the reason. It returns a file path, not inline image content. Screenshots + are not subject to the download byte limit. +- `browser-back` and `browser-forward` navigate the selected tab's history + and return a fresh snapshot. With no history entry, the page stays unchanged. +- `browser-reload` reloads the selected tab and returns a fresh snapshot. +- `browser-find "text"` searches current page text without case sensitivity, + returning surrounding excerpts and zero-based character offsets. Results + are limited to 50 matches and `playwrightMaxTextChars` excerpt characters; + truncated results are marked. No matches returns `NO_MATCHES`. +- `browser-read-more` returns the next `playwrightMaxTextChars` characters + from the latest captured text until `END_OF_TEXT`. Every fresh page snapshot + (including read, switch, click, scroll, and wait) restarts this cursor. + Continuation uses captured text, so asynchronous changes require a fresh + `browser-read`. Character ranges are zero-based with an exclusive end. +- `browser-links` lists up to 200 visible link labels and resolved destination + URLs, respecting the page's base URL. Truncation is marked. These URLs have + not been validated for navigation; the existing public-URL checks still + apply when opening them. This list does not assign click target numbers. +- `browser-click 3` clicks target 3 from the latest snapshot. +- `browser-type 3 "search terms"` replaces the contents of editable target 3 + and returns a short confirmation without reading the page or waiting for + the settling delay. Empty text clears the field. Supports native + text/search/email/password/URL/telephone/number inputs, textareas, and + contenteditable fields. After success, **only target 1 is valid**: it refers + to the exact element just filled. The function fills and verifies the + field's contents, allowing up to two retries after the initial attempt + (three attempts total). It stops early if the page closes or changes tabs, + or the field detaches. There is no keyboard fallback. Verification checks + the current value, not future site updates. + If the site replaces that element, obtain current targets with `browser-read`. + Use `browser-read` to inspect suggestions or find a search button. + Typing does not press Enter, though site input + handlers may trigger searches or other actions. Invalid targets, unsupported + field types, read-only/disabled fields, and execution errors return + `BROWSER-TYPE-FAILED`. Supplied text is omitted from fill error messages. + + On failure, do not repeat the text-entry request or press Enter. Inspect + with `browser-read` and report the failure; text may already be partially + entered. For searches, clicking the associated Search button is the default; + Enter is the fallback when no such button is available. + +- `browser-press-enter 3` presses Enter in editable target 3, waits for DOM + loading and the configured settling delay, and returns a fresh snapshot. + This command takes only a field number. In multiline fields, Enter may + insert a newline instead of submitting. Errors return + `BROWSER-PRESS-ENTER-FAILED`. + + Search workflow: + + 1. Find the search field marked `EDITABLE` in the latest snapshot. + 2. Call `browser-type` with that field number and the search text. + 3. Check for `BROWSER-TYPE-OK`. If typing failed, stop text entry, inspect + the page and report the failure. Do not retry typing or press Enter. + 4. Inspect the page with `browser-read`. If results already updated, do not + submit again. Otherwise, click the Search button associated with that + field using `browser-click`. If no such button is available, call + `browser-press-enter` on the search field. Use target numbers from this + fresh snapshot, not the numbers from before typing. + 5. Inspect the resulting page to confirm the search completed. If results + are still loading, use `browser-wait` before reading them again. + +- `browser-options 3` lists up to 200 options for native dropdown target 3, + including labels, values, disabled and selected states. Target numbers stay + unchanged. Snapshots identify dropdowns but do not enumerate their options. + +- `browser-select 3 "English"` selects an option in native dropdown target 3 + and returns a fresh snapshot. Use `browser-options` to see the choices first. + Matching is exact and + case-sensitive: labels take precedence over values. Missing, ambiguous, + disabled, or non-dropdown targets/options return `BROWSER-SELECT-FAILED`; + browser execution errors use the same prefix. Selection chooses one option + and replaces previous selections, including in multi-select controls. + Custom menus continue to use `browser-click`. Selection can trigger a site's + change handlers, including navigation or automatic submission. +- `browser-scroll 900` scrolls down 900 pixels and refreshes the snapshot; + negative values scroll upward. +- `browser-wait 1500` waits for asynchronous output and refreshes the snapshot. +- `browser-download 4` explicitly downloads target 4 to the dedicated download + directory and returns its verified byte count and SHA-256 digest. +- `browser-close` closes all tabs and destroys the browser context and its + session data. + +There is no plugin-imposed tab limit; capacity depends on available memory +and browser/OS resources. Read, click, scroll, wait, and download act on the +selected tab. Tab IDs remain stable until closed and reset after +`browser-close`. Click target numbers belong to the latest snapshot of the +selected tab; switching tabs refreshes them. Popups are also listed as tabs, +but do not automatically change the selection. + +Snapshots collect control descriptions in one browser call, with a maximum +of 200 controls. Dropdown options are read only by `browser-options` or during +selection. Typing skips the full snapshot and invalidates the previous text +reading; use `browser-read` before requesting `browser-read-more` again. + +Tabs share one browser context, including cookies and origin-scoped local +storage. Closing the last tab keeps that context alive; `browser-close` +discards it. Each `browser-open` now creates a tab; use `browser-navigate` +for the previous behavior of navigating the current page. + +Every browser command returns `BROWSER--FAILED: ` when +validation or execution fails, and logs the error. For example, reading +without a selected tab returns `BROWSER-READ-FAILED`, and an invalid tab ID +for `browser-close-tab` returns `BROWSER-CLOSE-TAB-FAILED`. A command may have +partially completed before failing; use `browser-tabs` and `browser-read` +to inspect the current state. After an exception, obtain a fresh snapshot +before using numbered click targets. + +The additional commands use `BROWSER-BACK-FAILED`, `BROWSER-FORWARD-FAILED`, +`BROWSER-RELOAD-FAILED`, `BROWSER-FIND-FAILED`, `BROWSER-READ-MORE-FAILED`, and +`BROWSER-LINKS-FAILED`. Empty link lists, no search matches, and end of text +are normal results, not failures. + +Standard controls and common custom click controls (`role`, `onclick`, and +`tabindex`) are numbered. A configurable settling delay allows client-rendered +output to appear after a click, Enter, selection, or scroll. Typing returns +immediately after filling; use `browser-wait` to inspect delayed suggestions. +There are no skills for +directly setting cookies, uploading files, +evaluating JavaScript, or restoring a prior browser profile. Downloads require +the explicit `browser-download` skill; downloads triggered by ordinary clicks +are cancelled. Local, +loopback, link-local, and private-network URLs are rejected, including requests +reached through redirects. + +## Enabling + +Install the Python dependency and the browser binary outside the agent, then +set: + +```yaml +playwrightEnabled: enabled +``` + +Optional settings are `playwrightBrowser` (`chromium`, `firefox`, or `webkit`), +`playwrightHeadless`, `playwrightTimeoutMs`, and `playwrightMaxTextChars`. +`playwrightSettleMs` controls the post-click/scroll delay and defaults to 750. +`playwrightDownloadDir` defaults to `memory/browser_downloads`, and +`playwrightMaxDownloadBytes` defaults to 10 MiB. Suggested filenames are +sanitized and existing files are never overwritten. + +The default is `enabled`. Set `playwrightEnabled: disabled` to +turn it off. Playwright browser binaries are not installed or downloaded +automatically by OmegaClaw. diff --git a/plugins/playwright/playwright.metta b/plugins/playwright/playwright.metta new file mode 100644 index 00000000..7603a1a8 --- /dev/null +++ b/plugins/playwright/playwright.metta @@ -0,0 +1,177 @@ +!(import! &self (library Omega ./plugins/playwright/playwright_bridge.py)) + +(= (playwrightEnabled) (empty)) +(= (playwrightBrowser) (empty)) +(= (playwrightHeadless) (empty)) +(= (playwrightTimeoutMs) (empty)) +(= (playwrightMaxTextChars) (empty)) +(= (playwrightSettleMs) (empty)) +(= (playwrightDownloadDir) (empty)) +(= (playwrightMaxDownloadBytes) (empty)) + +(= (loadOmegaPlugin) + (progn + (configure playwrightEnabled enabled) + (configure playwrightBrowser chromium) + (configure playwrightHeadless True) + (configure playwrightTimeoutMs 15000) + (configure playwrightMaxTextChars 20000) + (configure playwrightSettleMs 750) + (configure playwrightDownloadDir + (joinPath ((memoryDirectory) browser_downloads))) + (configure playwrightMaxDownloadBytes 10485760) + (if (== (playwrightEnabled) enabled) + (progn + (py-call (playwright_bridge.configure + (playwrightBrowser) + (playwrightHeadless) + (playwrightTimeoutMs) + (playwrightMaxTextChars) + (playwrightSettleMs) + (playwrightDownloadDir) + (playwrightMaxDownloadBytes))) + (add-skill browser-open + "Open a public HTTP(S) URL in a new selected tab in the shared browser session and return its tab ID, readable text, and numbered click targets" + (url_in_quotes)) + (add-skill browser-tabs + "List all open browser tab IDs and URLs, marking the selected tab; includes popups" + ()) + (add-skill browser-switch + "Select an open tab by its ID from browser-tabs and return a fresh page snapshot; use the new click target numbers" + (tab_id)) + (add-skill browser-close-tab + "Close a tab by ID and list remaining tabs; if selected, select the oldest remaining tab; call browser-read before clicking" + (tab_id)) + (add-skill browser-navigate + "Navigate the selected tab to a public HTTP(S) URL without opening a new tab and return a fresh page snapshot" + (url_in_quotes)) + (add-skill browser-read + "Read the current page and return its URL, title, visible text, and numbered click targets" + ()) + (add-skill browser-screenshot + (strings-concat + ("Save a full-page PNG screenshot of the selected tab into " + (playwrightDownloadDir) + "; by default the browser_downloads subdirectory of SAVE_PERMANENT_FILES_DIR. Returns the full saved path, byte count, and tab ID; errors return BROWSER-SCREENSHOT-FAILED")) + ()) + (add-skill browser-back + "Go back in the selected tab's history and return a fresh snapshot; unchanged if no previous entry" + ()) + (add-skill browser-forward + "Go forward in the selected tab's history and return a fresh snapshot; unchanged if no next entry" + ()) + (add-skill browser-reload + "Reload the selected tab and return a fresh page snapshot" + ()) + (add-skill browser-find + "Search the selected page's current text without case sensitivity and return matching excerpts with character offsets; at most 50 matches" + (text_in_quotes)) + (add-skill browser-read-more + "Return the next text chunk from the selected tab's latest captured snapshot until END_OF_TEXT; browser-read refreshes the text and restarts at the beginning" + ()) + (add-skill browser-links + "List up to 200 visible link labels and destination URLs in the selected tab; URLs are informational and not validated; use browser-read for click target numbers" + ()) + (add-skill browser-click + "Click a numbered target from the latest browser page snapshot and return the resulting page snapshot" + (target_number)) + (add-skill browser-select + "Select one option in a native dropdown using its target number and exact option label or value from browser-options; labels take precedence; replaces any prior selections and returns a fresh snapshot; errors return BROWSER-SELECT-FAILED" + (target_number option_in_quotes)) + (add-skill browser-options + "List up to 200 options for one native dropdown using its target number from browser-read. Shows labels, values, disabled and selected states without scanning the rest of the page. Target numbers are unchanged. Use browser-select to choose an option." + (target_number)) + (add-skill browser-type + "Fill a visible editable field and verify its contents; empty text clears it. Allows up to 2 retries after the initial attempt (3 total). Does not submit. Returns a brief confirmation; ONLY target 1 refers to the filled field until the next snapshot. Default search workflow: after success call browser-read; if results already updated, stop. Otherwise browser-click the Search button associated with that field; if none is available, browser-press-enter on the search field. Use the refreshed target numbers and inspect results afterward. On BROWSER-TYPE-FAILED do not start another retry cycle or submit: inspect and report the failure. Text may be partially entered." + (target_number text_in_quotes)) + (add-skill browser-press-enter + "Press Enter in an editable field. For search, first use browser-read and prefer clicking the Search button associated with the field; use Enter if no such button is available. Never submit after failed typing or if results already updated. Use the search field number from the latest snapshot (target 1 only immediately after browser-type). Waits for DOM loading and the settling delay, then returns a fresh snapshot; inspect results to confirm success. In multiline fields Enter may insert a newline. Errors return BROWSER-PRESS-ENTER-FAILED" + (target_number)) + (add-skill browser-scroll + "Scroll the current page up or down by a pixel amount, then return a refreshed page snapshot; positive numbers scroll down and negative numbers scroll up" + (pixel_amount)) + (add-skill browser-wait + "Wait briefly for asynchronous page content to update, then return a refreshed page snapshot; maximum 10000 milliseconds" + (milliseconds)) + (add-skill browser-download + (strings-concat + ("Download a numbered target from the selected tab's latest snapshot into " + (playwrightDownloadDir) + "; by default this is the browser_downloads subdirectory of SAVE_PERMANENT_FILES_DIR (the permanent files directory). Enforces the configured size limit and returns the full saved file path, byte count, and SHA-256 hash")) + (target_number)) + (add-skill browser-close + "Close and discard the playwright browser session, including all session cookies and storage" + ()) + (log INFO "playwright-plugin" "playwright read/click browser skills enabled") + ()) + (log INFO "playwright-plugin" "playwright read/click browser skills disabled")))) + +(= (browser-open $url) + (py-call (playwright_bridge.open_page $url))) + +(= (browser-tabs) + (py-call (playwright_bridge.list_tabs))) + +(= (browser-switch $tab) + (py-call (playwright_bridge.switch_tab $tab))) + +(= (browser-close-tab $tab) + (py-call (playwright_bridge.close_tab $tab))) + +(= (browser-navigate $url) + (py-call (playwright_bridge.navigate_page $url))) + +(= (browser-read) + (py-call (playwright_bridge.read_page))) + +(= (browser-screenshot) + (py-call (playwright_bridge.screenshot_page))) + +(= (browser-back) + (py-call (playwright_bridge.back_page))) + +(= (browser-forward) + (py-call (playwright_bridge.forward_page))) + +(= (browser-reload) + (py-call (playwright_bridge.reload_page))) + +(= (browser-find $text) + (py-call (playwright_bridge.find_text $text))) + +(= (browser-read-more) + (py-call (playwright_bridge.read_more))) + +(= (browser-links) + (py-call (playwright_bridge.list_links))) + +(= (browser-click $target) + (py-call (playwright_bridge.click_target $target))) + +(= (browser-select $target $option) + (py-call (playwright_bridge.select_dropdown $target $option))) + +(= (browser-options $target) + (py-call (playwright_bridge.list_options $target))) + +(= (browser-type $target $text) + (py-call (playwright_bridge.type_text $target $text))) + +; Accept the bot parser's combined field-number-and-text string. +(= (browser-type $request) + (py-call (playwright_bridge.type_text $request))) + +(= (browser-press-enter $target) + (py-call (playwright_bridge.press_enter $target))) + +(= (browser-scroll $pixels) + (py-call (playwright_bridge.scroll_page $pixels))) + +(= (browser-wait $milliseconds) + (py-call (playwright_bridge.wait_and_read $milliseconds))) + +(= (browser-download $target) + (py-call (playwright_bridge.download_target $target))) + +(= (browser-close) + (py-call (playwright_bridge.close_browser))) diff --git a/plugins/playwright/playwright_bridge.py b/plugins/playwright/playwright_bridge.py new file mode 100644 index 00000000..e8c981a1 --- /dev/null +++ b/plugins/playwright/playwright_bridge.py @@ -0,0 +1,895 @@ +# ----------------------------------------------------------------------------- +# Function: Execute Omega browser skill calls using Playwright, a browser automation +# library that requires a real browser (Chromium, Firefox, or WebKit) +# which can run headlessly without a visible window. +# Inputs: Skill calls with browser action arguments, plus browser settings. +# Outputs: Page snapshots, command results/errors, screenshots, and downloads. +# ----------------------------------------------------------------------------- + +import hashlib +import ipaddress +import json +import os +import re +import socket +import tempfile +from functools import wraps +from urllib.parse import urljoin, urlsplit + +from src.logger import get_logger + + +logger = get_logger(__name__) + +# Browser settings +_browser_name = "chromium" +_headless = True +_timeout_ms = 15_000 +_max_text_chars = 20_000 +_settle_ms = 750 +_download_dir = "" +_max_download_bytes = 10 * 1024 * 1024 +_explicit_download = None +_download_notice = "" + +# Browser session state +_playwright = None +_browser = None +_context = None +_page = None +_targets = [] +_tabs = {} +_next_tab_id = 1 +_reading = None + +############################################################################ +# Utility functions supporting browser skill calls: validation, session +# management, page snapshots, downloads, error handling, etc. +############################################################################ + +# Wrap browser commands with consistent error reporting for skill calls. +def _browser_command(command): + """Return skill-visible errors for validation and browser exceptions.""" + def decorate(function): + @wraps(function) + def wrapped(*args, **kwargs): + global _targets, _download_notice, _reading + try: + return function(*args, **kwargs) + except Exception as exc: + _clear_targets() + _reading = None + message = f"BROWSER-{command}-FAILED: {type(exc).__name__}: {exc}" + if command == "TYPE": + message += ( + "\nDo not retry this text-entry request or press Enter. " + "Text may already be partially entered. Use browser-read to " + "inspect the page and report the failure before taking another action." + ) + if _download_notice: + message += f"\n{_download_notice}" + _download_notice = "" + logger.error(f"[PLAYWRIGHT] {message}") + return message + return wrapped + return decorate + + +# Convert a setting value to a boolean using common true values. +def _as_bool(value): + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +# Validate and store browser settings and prepare the download directory. +def configure(browser_name="chromium", headless=True, timeout_ms=15000, + max_text_chars=20000, settle_ms=750, download_dir="", + max_download_bytes=10485760): + global _browser_name, _headless, _timeout_ms, _max_text_chars, _settle_ms + global _download_dir, _max_download_bytes + name = str(browser_name).strip().lower() + if name not in {"chromium", "firefox", "webkit"}: + raise ValueError("playwrightBrowser must be chromium, firefox, or webkit") + _browser_name = name + _headless = _as_bool(headless) + _timeout_ms = max(1000, int(timeout_ms)) + _max_text_chars = max(1000, int(max_text_chars)) + _settle_ms = min(10_000, max(0, int(settle_ms))) + _download_dir = os.path.realpath(str(download_dir).strip().strip('"')) + if not _download_dir: + raise ValueError("playwrightDownloadDir must not be empty") + os.makedirs(_download_dir, exist_ok=True) + _max_download_bytes = max(1, int(max_download_bytes)) + return True + + +# Validate an HTTP or HTTPS URL and reject credentials and non-public addresses. +def _validate_public_url(url): + value = str(url).strip().strip('"') + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"}: + raise ValueError("only http:// and https:// URLs are allowed") + if not parsed.hostname or parsed.username or parsed.password: + raise ValueError("URL must have a hostname and must not contain credentials") + if parsed.hostname.lower() == "localhost": + raise ValueError("local and private network URLs are not allowed") + + # Resolve before navigation to reject obvious SSRF targets. The request + # interception below repeats this check for redirects and subresources. + try: + addresses = {item[4][0] for item in socket.getaddrinfo(parsed.hostname, parsed.port)} + except socket.gaierror as exc: + raise ValueError(f"hostname could not be resolved: {parsed.hostname}") from exc + if not addresses: + raise ValueError("hostname did not resolve") + for address in addresses: + ip = ipaddress.ip_address(address) + if not ip.is_global: + raise ValueError("local and private network URLs are not allowed") + return value + + +# Allow browser requests only when their URLs pass public-address validation. +def _route_request(route): + try: + _validate_public_url(route.request.url) + route.continue_() + except Exception: + logger.warning("[PLAYWRIGHT] blocked non-public request") + route.abort("blockedbyclient") + + +# Return the browser session, starting Playwright and the browser if needed. +def _ensure_context(): + global _playwright, _browser, _context + if _context is not None: + return _context + + # Import lazily so a disabled plugin does not require browser + # binaries or initialize Playwright during normal OmegaClaw startup. + from playwright.sync_api import sync_playwright + + _playwright = sync_playwright().start() + browser_type = getattr(_playwright, _browser_name) + _browser = browser_type.launch(headless=_headless) + _context = _browser.new_context( + accept_downloads=True, + service_workers="block", + ) + _context.set_default_timeout(_timeout_ms) + _context.set_default_navigation_timeout(_timeout_ms) + _context.route("**/*", _route_request) + _context.on("page", _register_page) + return _context + + +# Track a new tab and attach dialog, download, and close handlers. +def _register_page(page): + """Track explicit tabs and popups without changing the selected tab.""" + global _next_tab_id + if page in _tabs.values(): + return + tab_id = _next_tab_id + _next_tab_id += 1 + _tabs[tab_id] = page + page.on("dialog", lambda dialog: dialog.dismiss()) + page.on("download", _handle_download) + page.on("close", lambda: _forget_page(tab_id)) + + +# Remove a closed tab and select a remaining tab when necessary. +def _forget_page(tab_id): + global _page, _targets, _reading + closed = _tabs.pop(tab_id, None) + if closed is _page: + _page = next(iter(_tabs.values()), None) + _clear_targets() + _reading = None + + +# Select a tab and clear the previous interaction targets and saved text. +def _select_page(page): + global _page, _targets, _reading + _page = page + _clear_targets() + _reading = None + + +# Format the open tab IDs and URLs and mark the selected tab. +def _tab_list(): + lines = ["BROWSER-TABS"] + for tab_id, page in list(_tabs.items()): + selected = " (selected)" if page is _page else "" + lines.append(f"[{tab_id}]{selected} {page.url}") + return "\n".join(lines) if _tabs else "BROWSER-TABS: (none)" + + +# Look up a tab by ID, returning None if the ID is invalid or unknown. +def _find_tab(tab_id): + try: + return _tabs.get(int(str(tab_id).strip())) + except ValueError: + return None + + +# Cancel downloads that were not explicitly requested and record a notice. +def _handle_download(download): + """Cancel downloads unless browser-download deliberately initiated one.""" + global _download_notice + if download.page is not _explicit_download: + _download_notice = ( + "BROWSER-DOWNLOAD-BLOCKED: automatic or ordinary-click download " + "was cancelled. Use browser-download with a target number from " + "the selected tab's latest snapshot to save a file." + ) + logger.warning(f"[PLAYWRIGHT] {_download_notice}") + try: + download.cancel() + except Exception as exc: + _download_notice += f" Cancellation failed: {type(exc).__name__}: {exc}" + logger.error(f"[PLAYWRIGHT] {_download_notice}") + + +# Collapse whitespace in text into single spaces and trim its ends. +def _clean(value): + return " ".join((value or "").split()) + + +# Clear interaction targets and release any pinned element handles. +def _clear_targets(): + """Release any pinned typing field before replacing the target list.""" + global _targets + previous = _targets + _targets = [] + for target in previous: + if hasattr(target, "dispose"): + try: + target.dispose() + except Exception: + pass # A closed page may already have released its handles. + + +# Capture page text and numbered interaction targets, saving remaining text for later reads. +def _snapshot(): + global _targets, _download_notice, _reading + _clear_targets() + _reading = None + if _page is None: + raise RuntimeError("no selected tab; call browser-open first") + + body = _page.locator("body") + text = body.inner_text(timeout=_timeout_ms) if body.count() else "" + full_text = text + text = full_text[:_max_text_chars] + + # Include common custom controls used by client-rendered documentation and + # notebook sites, along with native text fields and editable content. + candidates = _page.locator( + "a:visible, button:visible, summary:visible, select:visible, " + "[role='button']:visible, [role='link']:visible, " + "[role='menuitem']:visible, [onclick]:visible, [tabindex]:visible, " + "input[type='button']:visible, input[type='submit']:visible, " + "input:not([type]):visible, input[type='text' i]:visible, " + "input[type='search' i]:visible, input[type='email' i]:visible, " + "input[type='password' i]:visible, input[type='tel' i]:visible, " + "input[type='url' i]:visible, input[type='number' i]:visible, " + "textarea:visible, [contenteditable='true' i]:visible, " + "[contenteditable='']:visible, [contenteditable='plaintext-only' i]:visible" + ) + # Read metadata in one browser call, rather than several calls per control. + controls = candidates.evaluate_all(r"""elements => { + const clean = value => (value || '').replace(/\s+/g, ' ').trim(); + return {count: elements.length, items: elements.slice(0, 200).map(el => { + const field = el.matches('input, textarea'); + const textField = el.matches('textarea') || (el.matches('input') && + ['text', 'search', 'email', 'password', 'tel', 'url', 'number'].includes(el.type)); + const dropdown = el.matches('select'); + const disabled = el.matches(':disabled') || + !!el.closest('[aria-disabled="true"]'); + const label = clean(field || dropdown ? '' : el.innerText) || + clean(el.getAttribute('aria-label')) || clean(el.getAttribute('title')) || + clean(el.getAttribute('placeholder')) || clean(el.getAttribute('name')) || + clean(field ? '' : el.getAttribute('value')) || 'unlabelled control'; + return {label: label.slice(0, 240), dropdown, disabled, + editable: (textField || el.isContentEditable) && !disabled && + !el.readOnly && el.getAttribute('aria-readonly') !== 'true'}; + })}; + }""") + lines = [] + for index, control in enumerate(controls["items"]): + _targets.append(candidates.nth(index)) + lines.append(f"[{index + 1}] {control['label']}") + if control["editable"]: + lines.append(" EDITABLE: browser-type fills; for search, click the associated Search button or use Enter if none") + if control["dropdown"]: + state = "disabled" if control["disabled"] else "enabled" + lines.append(f" DROPDOWN ({state}): browser-options lists choices; browser-select selects") + if controls["count"] > 200: + lines.append(f"TARGETS_TRUNCATED: showing 200 of {controls['count']}") + + targets = "\n".join(lines) if lines else "(none)" + notice = _download_notice + _download_notice = "" + result = ( + f"BROWSER-PAGE\nTAB: {next(key for key, page in _tabs.items() if page is _page)}\n" + f"URL: {_page.url}\nTITLE: {_clean(_page.title())}\n" + f"TEXT:\n{text}\nTEXT_CHARS: {len(text)}/{len(full_text)}" + + ("\nMORE_TEXT: call browser-read-more" if len(text) < len(full_text) else "\nEND_OF_TEXT") + + f"\nCLICK_TARGETS:\n{targets}" + + (f"\n{notice}" if notice else "") + ) + _reading = (_page, _page.url, full_text, len(text)) + return result + + +# Return the selected open tab or raise an error if none is available. +def _require_page(): + if _page is None or _page.is_closed(): + raise RuntimeError("no selected open tab; call browser-open first") + return _page + + +# Perform a history or reload action and return the updated page snapshot. +def _history_action(method): + page = _require_page() + _select_page(page) + # A missing history entry is a harmless no-op; same-document history can + # also return no response, so do not infer failure from a None response. + getattr(page, method)(wait_until="domcontentloaded", timeout=_timeout_ms) + if _settle_ms: + page.wait_for_timeout(_settle_ms) + return _snapshot() + + +# Sanitize a suggested download filename and limit its length. +def _safe_download_name(suggested): + name = os.path.basename(str(suggested or "download")) + name = re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._") + if not name: + name = "download" + return name[:180] + + +# Choose an unused file path within the configured download directory. +def _unused_download_path(filename): + stem, extension = os.path.splitext(filename) + candidate = os.path.join(_download_dir, filename) + number = 1 + while os.path.exists(candidate): + candidate = os.path.join(_download_dir, f"{stem}-{number}{extension}") + number += 1 + candidate = os.path.realpath(candidate) + if os.path.commonpath((_download_dir, candidate)) != _download_dir: + raise ValueError("unsafe download filename") + return candidate + + +# Log and return a download failure message with the download directory. +def _download_error(reason): + message = f"BROWSER-DOWNLOAD-FAILED: {reason}; download directory: {_download_dir}" + logger.error(f"[PLAYWRIGHT] {message}") + return message + + +############################################################################ +# Browser commands invoked by skill calls. +############################################################################ + +# List open tabs and identify the selected tab. +@_browser_command("TABS") +def list_tabs(): + return _tab_list() + + +# Switch to the specified tab and return its page snapshot. +@_browser_command("SWITCH") +def switch_tab(tab_id): + page = _find_tab(tab_id) + if page is None: + raise ValueError("use a tab ID from browser-tabs") + _select_page(page) + return _snapshot() + + +# Close the specified tab and list the remaining tabs. +@_browser_command("CLOSE-TAB") +def close_tab(tab_id): + page = _find_tab(tab_id) + if page is None: + raise ValueError("use a tab ID from browser-tabs") + page.close() + return _tab_list() + + +# Open a public URL in a new tab and return its page snapshot. +@_browser_command("OPEN") +def open_page(url): + safe_url = _validate_public_url(url) + context = _ensure_context() + previous = _page + page = context.new_page() + _register_page(page) + _select_page(page) + logger.info("[PLAYWRIGHT] opening public page") + try: + page.goto(safe_url, wait_until="domcontentloaded") + except Exception: + page.close() + if previous in _tabs.values(): + _select_page(previous) + raise + return _snapshot() + + +# Navigate the selected tab to a public URL and return its page snapshot. +@_browser_command("NAVIGATE") +def navigate_page(url): + if _page is None: + raise RuntimeError("no selected tab; call browser-open first") + safe_url = _validate_public_url(url) + _select_page(_page) + _page.goto(safe_url, wait_until="domcontentloaded") + return _snapshot() + + +# Return the selected page's text and numbered interaction targets. +@_browser_command("READ") +def read_page(): + return _snapshot() + + +# Save a full-page screenshot of the selected tab. +@_browser_command("SCREENSHOT") +def screenshot_page(): + page = _require_page() + if not _download_dir: + raise RuntimeError("download directory is not configured") + os.makedirs(_download_dir, exist_ok=True) + tab_id = next(key for key, tab in _tabs.items() if tab is page) + # Reserve a unique filename so previous screenshots are never overwritten. + descriptor, destination = tempfile.mkstemp( + prefix=f"screenshot-tab-{tab_id}-", suffix=".png", dir=_download_dir + ) + os.close(descriptor) + try: + page.screenshot(path=destination, full_page=True, timeout=_timeout_ms) + size = os.path.getsize(destination) + if size == 0: + raise RuntimeError("browser produced an empty screenshot") + except Exception: + os.unlink(destination) + raise + logger.info(f"[PLAYWRIGHT] screenshot saved file={destination} bytes={size}") + return f"BROWSER-SCREENSHOT-SAVED file={destination} bytes={size} tab={tab_id}" + + +# Go back in the selected tab's history and return its page snapshot. +@_browser_command("BACK") +def back_page(): + return _history_action("go_back") + + +# Go forward in the selected tab's history and return its page snapshot. +@_browser_command("FORWARD") +def forward_page(): + return _history_action("go_forward") + + +# Reload the selected tab and return its page snapshot. +@_browser_command("RELOAD") +def reload_page(): + return _history_action("reload") + + +# Return the next portion of text from the saved page snapshot. +@_browser_command("READ-MORE") +def read_more(): + global _reading + page = _require_page() + if _reading is None or _reading[0] is not page or _reading[1] != page.url: + raise RuntimeError("no text snapshot for this page; call browser-read first") + _, url, text, start = _reading + end = min(start + _max_text_chars, len(text)) + _reading = (page, url, text, end) + status = "MORE_TEXT: call browser-read-more" if end < len(text) else "END_OF_TEXT" + return ( + f"BROWSER-READ-MORE\nURL: {url}\nTEXT_RANGE: {start}:{end}/{len(text)}\n" + f"TEXT:\n{text[start:end]}\n{status}" + ) + + +# Search the selected page's text and return matching excerpts. +@_browser_command("FIND") +def find_text(query): + page = _require_page() + query = str(query).strip().strip('"') + if not query: + raise ValueError("search text must not be empty") + body = page.locator("body") + text = body.inner_text(timeout=_timeout_ms) if body.count() else "" + lines = [] + used = 0 + truncated = False + for match in re.finditer(re.escape(query), text, flags=re.IGNORECASE): + excerpt = _clean(text[max(0, match.start() - 160):match.end() + 160]) + line = f"[{match.start()}] {excerpt}" + remaining = _max_text_chars - used + if len(lines) >= 50 or remaining <= 0: + truncated = True + break + lines.append(line[:remaining]) + used += len(line) + 1 + if len(line) > remaining: + truncated = True + break + result = "\n".join(lines) if lines else "NO_MATCHES" + if truncated: + result += "\nRESULTS_TRUNCATED: use a more specific search" + return f"BROWSER-FIND\nURL: {page.url}\n{result}" + + +# List visible links and their destinations on the selected page. +@_browser_command("LINKS") +def list_links(): + page = _require_page() + base = page.url + bases = page.locator("base[href]") + if bases.count(): + base = urljoin(base, bases.nth(0).get_attribute("href") or "") + links = page.locator("a[href]:visible, area[href]:visible") + count = links.count() + lines = [] + for index in range(min(count, 200)): + link = links.nth(index) + label = _clean( + link.inner_text(timeout=_timeout_ms) + or link.get_attribute("aria-label") + or link.get_attribute("title") + or link.get_attribute("alt") + or "unlabelled link" + ) + destination = urljoin(base, link.get_attribute("href") or "") + lines.append(f"- {label[:240]} -> {destination}") + result = "\n".join(lines) if lines else "(none)" + if count > 200: + result += f"\nLINKS_TRUNCATED: showing 200 of {count}" + return ( + f"BROWSER-LINKS\nURL: {page.url}\n{result}\n" + "Destinations are informational, not validated. Use browser-read for click target numbers." + ) + + +# Click a numbered target and return the updated page snapshot. +@_browser_command("CLICK") +def click_target(target): + if _page is None: + raise RuntimeError("no selected tab; call browser-open first") + try: + index = int(str(target).strip()) - 1 + except ValueError: + raise ValueError("target must be a number from the latest snapshot") from None + if index < 0 or index >= len(_targets): + raise ValueError("target is not present in the latest snapshot; call browser-read first") + + _targets[index].click(timeout=_timeout_ms) + if _page is None: + raise RuntimeError("selected tab closed during click; use browser-tabs or browser-open") + _page.wait_for_load_state("domcontentloaded", timeout=_timeout_ms) + if _settle_ms and _page is not None: + _page.wait_for_timeout(_settle_ms) + return _snapshot() + + +# Fill an editable target with text and verify its contents. +@_browser_command("TYPE") +def type_text(target, text=None): + global _targets, _reading, _download_notice + # The bot parser combines the field number and text into one string. + # Keep accepting separate arguments for direct calls as well. + if text is None: + match = re.fullmatch(r'\s*(?:"([0-9]+)"|([0-9]+))\s+([\s\S]*)', str(target)) + if match is None: + raise ValueError('expected a field number followed by text, for example: 11 "search terms"') + target = match.group(1) or match.group(2) + text = match.group(3) + if text.startswith('"'): + try: + text = json.loads(text) + except ValueError: + raise ValueError("text has invalid quoting; use a double-quoted string") from None + page = _require_page() + try: + index = int(str(target).strip()) - 1 + except ValueError: + raise ValueError("target must be a field number from the latest snapshot") from None + if index < 0 or index >= len(_targets): + raise ValueError("target is not present in the latest snapshot; call browser-read first") + field = _targets[index] + value = str(text) + _reading = None + stage = "resolving the field" + attempts = 0 + # Pin the element so validation, filling and + # verification cannot silently switch controls after a DOM update. + try: + if hasattr(field, "element_handle"): + field = field.element_handle(timeout=_timeout_ms) + _clear_targets() + if field is None: + raise ValueError("field is no longer present") + _targets = [field] + stage = "checking the field is visible and editable" + if not field.is_visible() or not field.is_editable(): + raise ValueError("field is hidden, disabled, read-only, or not editable") + for attempts in range(1, 4): # Initial attempt plus at most two retries. + try: + stage = "filling the field" + field.fill(value, timeout=_timeout_ms) + stage = "checking the selected tab after filling" + if page.is_closed() or _page is not page: + raise RuntimeError("selected tab changed or closed") + stage = "verifying that the field contains the requested text" + matches = field.evaluate(r"""(el, expected) => { + if (!el.isConnected) return false; + if (el.matches('input, textarea')) { + if (el.matches('textarea')) expected = expected.replace(/\r\n?/g, '\n'); + return el.value === expected; + } + if (el.isContentEditable) { + const normalize = value => value.replace(/\r\n?/g, '\n'); + return normalize(el.innerText) === normalize(expected); + } + return false; + }""", value) + if not matches: + raise ValueError("field contents differ or the field was replaced") + break + except Exception: + if attempts == 3 or page.is_closed() or _page is not page: + raise + if not field.evaluate("el => el.isConnected"): + raise + except Exception as exc: + # Playwright call logs can include supplied text. Report only the + # failed stage and exception class, never the raw exception or values. + raise RuntimeError( + f"{stage} failed ({type(exc).__name__}); {attempts} fill attempt(s), " + "at most 2 retries allowed" + ) from None + notice = _download_notice + _download_notice = "" + return ( + "BROWSER-TYPE-OK: field contents verified\n[1] Field just filled\n" + "No Enter was pressed. Previous target numbers are invalid. " + "For search, call browser-read. If results already updated, do not submit again. " + "Otherwise click the Search button associated with this field; if none is available, " + "use browser-press-enter on the search field. Use target numbers from the new snapshot " + "and inspect results afterward. Target 1 refers to this field only until that snapshot." + + (f"\n{notice}" if notice else "") + ) + + +# Press Enter in an editable target and return the updated page snapshot. +@_browser_command("PRESS-ENTER") +def press_enter(target): + page = _require_page() + try: + index = int(str(target).strip()) - 1 + except ValueError: + raise ValueError("target must be a field number from the latest snapshot") from None + if index < 0 or index >= len(_targets): + raise ValueError("target is not present in the latest snapshot; call browser-read first") + field = _targets[index] + if not field.is_editable(): + raise ValueError("target is not editable or is disabled/read-only") + field.press("Enter", timeout=_timeout_ms) + if page.is_closed() or _page is not page: + raise RuntimeError("selected tab closed after pressing Enter; use browser-tabs") + page.wait_for_load_state("domcontentloaded", timeout=_timeout_ms) + if _settle_ms: + page.wait_for_timeout(_settle_ms) + return _snapshot() + + +# List the choices available in a native dropdown target. +@_browser_command("OPTIONS") +def list_options(target): + _require_page() + try: + index = int(str(target).strip()) - 1 + except ValueError: + raise ValueError("target must be a dropdown number from the latest snapshot") from None + if index < 0 or index >= len(_targets): + raise ValueError("target is not present in the latest snapshot; call browser-read first") + data = _targets[index].evaluate("""el => { + if (el.tagName !== 'SELECT') return null; + return {count: el.options.length, + options: Array.from(el.options).slice(0, 200).map(option => ({ + label: option.label, value: option.value, selected: option.selected, + disabled: el.matches(':disabled') || option.matches(':disabled') + }))}; + }""") + if data is None: + raise ValueError("target is not a native dropdown; use browser-click for custom menus") + lines = [f"BROWSER-OPTIONS target={index + 1}"] + for option in data["options"]: + flags = [name for name in ("disabled", "selected") if option[name]] + lines.append( + f"label={option['label']!r} value={option['value']!r}" + + (f" ({', '.join(flags)})" if flags else "") + ) + if data["count"] > 200: + lines.append("OPTIONS_TRUNCATED: first 200 options shown") + return "\n".join(lines) + + +# Select a native dropdown option by its label or value. +@_browser_command("SELECT") +def select_dropdown(target, choice): + page = _require_page() + try: + index = int(str(target).strip()) - 1 + except ValueError: + raise ValueError("target must be a dropdown number from the latest snapshot") from None + if index < 0 or index >= len(_targets): + raise ValueError("target is not present in the latest snapshot; call browser-read first") + dropdown = _targets[index] + if not dropdown.evaluate("el => el.tagName === 'SELECT'"): + raise ValueError("target is not a native dropdown; use browser-click for custom menus") + if dropdown.is_disabled(): + raise ValueError("dropdown is disabled") + choice = str(choice) + if len(choice) >= 2 and choice[0] == choice[-1] == '"': + choice = choice[1:-1] + options = dropdown.locator("option") + label_matches = [] + value_matches = [] + for option_index in range(options.count()): + option = options.nth(option_index) + label = option.get_attribute("label") + text = _clean(option.text_content(timeout=_timeout_ms)) + if label is None: + label = text + value = option.get_attribute("value") + if value is None: + value = text + if label == choice: + label_matches.append(option_index) + if value == choice: + value_matches.append(option_index) + matches = label_matches or value_matches + if not matches: + raise ValueError("option not found; use an exact label or value from browser-options") + if len(matches) != 1: + raise ValueError("option is ambiguous; use a unique option label or value") + if options.nth(matches[0]).is_disabled(): + raise ValueError("option is disabled") + dropdown.select_option(index=matches[0], timeout=_timeout_ms) + if page.is_closed() or _page is not page: + raise RuntimeError("selected tab closed during selection; use browser-tabs") + if _settle_ms: + page.wait_for_timeout(_settle_ms) + return _snapshot() + + +# Scroll the selected page vertically and return its page snapshot. +@_browser_command("SCROLL") +def scroll_page(pixels): + if _page is None: + raise RuntimeError("no selected tab; call browser-open first") + try: + amount = int(str(pixels).strip()) + except ValueError: + raise ValueError("pixel amount must be an integer") from None + amount = min(5000, max(-5000, amount)) + _page.mouse.wheel(0, amount) + if _settle_ms: + _page.wait_for_timeout(_settle_ms) + return _snapshot() + + +# Wait for the specified duration and return the page snapshot. +@_browser_command("WAIT") +def wait_and_read(milliseconds): + if _page is None: + raise RuntimeError("no selected tab; call browser-open first") + try: + duration = int(str(milliseconds).strip()) + except ValueError: + raise ValueError("milliseconds must be an integer") from None + duration = min(10_000, max(0, duration)) + _page.wait_for_timeout(duration) + return _snapshot() + + +# Click a target to download a file, enforce the size limit, and verify the saved file. +@_browser_command("DOWNLOAD") +def download_target(target): + global _explicit_download + if _page is None: + return _download_error("no selected tab; call browser-open first") + try: + index = int(str(target).strip()) - 1 + except ValueError: + return _download_error("target must be a number from the latest snapshot") + if index < 0 or index >= len(_targets): + return _download_error("target is not present in the latest snapshot; call browser-read first") + + stage = "waiting for a download event after clicking the target" + try: + logger.info( + f"[PLAYWRIGHT] download starting target={target} directory={_download_dir}" + ) + _explicit_download = _page + with _page.expect_download(timeout=_timeout_ms) as pending: + _targets[index].click(timeout=_timeout_ms) + download = pending.value + stage = "waiting for the download to complete" + failure = download.failure() + if failure: + return _download_error(f"browser reported: {failure}") + stage = "checking the downloaded temporary file" + temporary_path = download.path() + if temporary_path is None: + return _download_error("browser returned no temporary file") + size = os.path.getsize(temporary_path) + if size > _max_download_bytes: + download.cancel() + return _download_error( + f"file is {size} bytes; limit is " + f"{_max_download_bytes} bytes" + ) + + destination = _unused_download_path( + _safe_download_name(download.suggested_filename) + ) + stage = f"saving file to {destination}" + download.save_as(destination) + stage = f"verifying saved file {destination}" + with open(destination, "rb") as downloaded: + digest = hashlib.sha256(downloaded.read()).hexdigest() + verified_size = os.path.getsize(destination) + logger.info( + f"[PLAYWRIGHT] download ok file={destination} bytes={verified_size} " + f"sha256={digest[:16]}" + ) + return ( + f"BROWSER-DOWNLOAD-VERIFIED file={destination} " + f"bytes={verified_size} sha256={digest}" + ) + except Exception as exc: + return _download_error(f"{stage}: {type(exc).__name__}: {exc}") + finally: + _explicit_download = None + + +# Close the browser session and clear its tracked state. +@_browser_command("CLOSE") +def close_browser(): + global _playwright, _browser, _context, _page, _targets, _explicit_download + global _next_tab_id, _download_notice, _reading + failures = [] + try: + for name, resource, method in ( + ("context", _context, "close"), + ("browser", _browser, "close"), + ("playwright", _playwright, "stop"), + ): + if resource is not None: + try: + getattr(resource, method)() + except Exception as exc: + failures.append(f"{name}: {type(exc).__name__}: {exc}") + finally: + _playwright = None + _browser = None + _context = None + _page = None + _clear_targets() + _explicit_download = None + _download_notice = "" + _tabs.clear() + _next_tab_id = 1 + _reading = None + if failures: + raise RuntimeError("; ".join(failures)) + return "BROWSER-CLOSED: session cookies and storage discarded" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 19f8bd16..194356a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ py-landlock==0.1.1 pyyaml==6.0.3 ddgs==9.14.4 websockets==16.0 +playwright==1.63.0 diff --git a/scripts/omega b/scripts/omega index bcd67b7b..cf5ff167 100755 --- a/scripts/omega +++ b/scripts/omega @@ -801,7 +801,8 @@ start() { --security-opt no-new-privileges:true --init --add-host=host.docker.internal:host-gateway - --tmpfs /tmp:size=64m,mode=1777 + --shm-size=1g + --tmpfs /tmp:size=1g,mode=1777 --tmpfs /var/tmp:size=64m,mode=1777 --tmpfs /run:size=16m,mode=755 --volume omega-memory:/PeTTa/repos/Omega/memory