diff --git a/griptape/drivers/web_scraper/markdownify_web_scraper_driver.py b/griptape/drivers/web_scraper/markdownify_web_scraper_driver.py index 5a302565cb..0abdf26ca2 100644 --- a/griptape/drivers/web_scraper/markdownify_web_scraper_driver.py +++ b/griptape/drivers/web_scraper/markdownify_web_scraper_driver.py @@ -25,6 +25,9 @@ class MarkdownifyWebScraperDriver(BaseWebScraperDriver): exclude_ids: Optionally provide custom ids to exclude from the scraped content. timeout: Optionally provide a timeout in milliseconds for the page to continue loading after the browser has emitted the "load" event. + browser_type: Optionally provide a Playwright browser type (chromium, firefox, webkit). + Defaults to chromium. + executable_path: Optionally provide a path to a custom browser executable. """ DEFAULT_EXCLUDE_TAGS = ["script", "style", "head", "audio", "img", "picture", "source", "video"] @@ -37,11 +40,17 @@ class MarkdownifyWebScraperDriver(BaseWebScraperDriver): exclude_classes: list[str] = field(default=Factory(list), kw_only=True) exclude_ids: list[str] = field(default=Factory(list), kw_only=True) timeout: int | None = field(default=None, kw_only=True) + browser_type: str = field(default="chromium", kw_only=True) + executable_path: str | None = field(default=None, kw_only=True) def fetch_url(self, url: str) -> str: sync_playwright = import_optional_dependency("playwright.sync_api").sync_playwright - with sync_playwright() as p, p.chromium.launch(headless=True) as browser: + launch_kwargs: dict[str, Any] = {"headless": True} + if self.executable_path is not None: + launch_kwargs["executable_path"] = self.executable_path + + with sync_playwright() as p, getattr(p, self.browser_type).launch(**launch_kwargs) as browser: page = browser.new_page() def skip_loading_images(route: Any) -> Any: diff --git a/tests/unit/drivers/web_scraper/test_markdownify_web_scraper_driver.py b/tests/unit/drivers/web_scraper/test_markdownify_web_scraper_driver.py index dbdafa98f5..2ee1301a84 100644 --- a/tests/unit/drivers/web_scraper/test_markdownify_web_scraper_driver.py +++ b/tests/unit/drivers/web_scraper/test_markdownify_web_scraper_driver.py @@ -88,3 +88,22 @@ def test_scrape_url_raises_on_none_from_playwright(self, web_scraper, mock_conte with pytest.raises(Exception, match="can't access URL"): web_scraper.scrape_url("https://example.com/") + + def test_default_launch_uses_chromium_headless(self, web_scraper, mock_playwright): + web_scraper.scrape_url("https://example.com/") + mock_playwright.__enter__.return_value.chromium.launch.assert_called_once_with(headless=True) + + def test_executable_path_passthrough(self, mock_playwright, mock_content): + MarkdownifyWebScraperDriver(executable_path="/opt/ff/firefox").scrape_url("https://example.com/") + mock_playwright.__enter__.return_value.chromium.launch.assert_called_once_with( + headless=True, executable_path="/opt/ff/firefox" + ) + + def test_browser_type_selects_firefox(self, mock_playwright): + firefox_launch = mock_playwright.__enter__.return_value.firefox.launch + firefox_launch.return_value.__enter__.return_value.new_page.return_value.content.return_value = ( + 'foobar' + ) + artifact = MarkdownifyWebScraperDriver(browser_type="firefox").scrape_url("https://example.com/") + assert artifact.value == "[foobar](foobar.com)" + firefox_launch.assert_called_once_with(headless=True)