Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 38 additions & 6 deletions src/wordstat/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,22 @@
from wordstat.collector import WordstatCollector
from wordstat.config import load_config
from wordstat.errors import WordstatError
from wordstat.periods import Granularity, parse_date, validate_period
from wordstat.storage import prepare_resume_directory

_CONFIG = load_config()
_DEFAULT_CDP_URL = _CONFIG.get("cdp_url", "http://127.0.0.1:9222")
# Wordstat's native export encoding is cp1251; a phrases file typed or saved
# on the same machine can plausibly be in either. Mirrors the encoding probe
# in csv_io.py, minus utf-8-sig (a phrases file is authored by hand, not
# exported by Wordstat, so a BOM is unlikely but harmless either way).
_PHRASES_FILE_ENCODINGS = ("utf-8", "cp1251")
# Wordstat exports CSV as UTF-8 with BOM; a phrases file typed or saved
# on the same machine can plausibly be in either, and utf-8-sig must come
# before plain utf-8: a BOM'd file decodes successfully under plain
# "utf-8" too, but leaves "" attached to the first line. Neither
# resolve_phrases' line.strip() nor the collector's own phrase.strip()
# removes it (''.isspace() is False), so a BOM left in is not
# harmless — it silently prepends an invisible character to the first
# phrase, which then propagates into the typed Wordstat search, the
# run-directory slug, and manifest.json. Mirrors the encoding probe in
# csv_io.py.
_PHRASES_FILE_ENCODINGS = ("utf-8-sig", "cp1251")


@click.group()
Expand Down Expand Up @@ -69,6 +76,14 @@ def _read_phrases_file(path: Path) -> str:
)
@click.option("--cdp-url", envvar="WORDSTAT_CDP_URL", default=_DEFAULT_CDP_URL, show_default=True)
@click.option("--timeout", "timeout_seconds", type=click.FloatRange(min=1), default=45.0, show_default=True)
@click.option(
"--granularity",
type=click.Choice(Granularity, case_sensitive=False),
default=Granularity.MONTHLY,
show_default=True,
)
@click.option("--date-from", type=str, default=None, help="Dynamics window start (YYYY-MM-DD).")
@click.option("--date-to", type=str, default=None, help="Dynamics window end (YYYY-MM-DD).")
@click.option(
"--keep-raw",
is_flag=True,
Expand All @@ -95,6 +110,9 @@ def collect(
output_dir: Path,
cdp_url: str,
timeout_seconds: float,
granularity: str,
date_from: str | None,
date_to: str | None,
keep_raw: bool,
resume_dir: Path | None,
) -> None:
Expand All @@ -106,6 +124,13 @@ def collect(
"""

phrases = resolve_phrases(phrase, phrases_file)
selected_granularity = Granularity(granularity.lower())
try:
parsed_from = parse_date(date_from)
parsed_to = parse_date(date_to)
validate_period(selected_granularity, parsed_from, parsed_to)
except WordstatError as error:
raise click.ClickException(str(error)) from error
if not phrases:
raise click.ClickException("At least one search phrase is required")
if resume_dir is not None:
Expand All @@ -130,7 +155,14 @@ def collect(
keep_raw=keep_raw,
)
try:
batch = asyncio.run(collector.collect_many(phrases, region=region, resume_directory=resume_dir))
collect_kwargs = {"region": region, "resume_directory": resume_dir}
if selected_granularity is not Granularity.MONTHLY or parsed_from is not None:
collect_kwargs.update(
granularity=selected_granularity,
date_from=parsed_from,
date_to=parsed_to,
)
batch = asyncio.run(collector.collect_many(phrases, **collect_kwargs))
# Only domain errors become friendly messages; an unexpected ValueError
# from a dependency should keep its traceback instead of being reworded.
except WordstatError as error:
Expand Down
433 changes: 420 additions & 13 deletions src/wordstat/collector.py

Large diffs are not rendered by default.

8 changes: 5 additions & 3 deletions src/wordstat/dataset_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
from wordstat.models import CsvDataset


def write_dataset(dataset: CsvDataset, run_directory: Path) -> tuple[Path, dict[str, str]]:
"""Write one report as ``<view>.parquet`` and report the inferred column types.
def write_dataset(
dataset: CsvDataset, run_directory: Path, file_name: str | None = None
) -> tuple[Path, dict[str, str]]:
"""Write one report and report the inferred column types.

Column order follows the export's own header order. The returned dtype
mapping goes into the manifest so an unexpected export format is visible
Expand All @@ -33,6 +35,6 @@ def write_dataset(dataset: CsvDataset, run_directory: Path) -> tuple[Path, dict[
columns[header] = pa.array(coerced, type=pa.type_for_alias(dtype))

table = pa.table(columns)
destination = run_directory / f"{dataset.view.value}.parquet"
destination = run_directory / (file_name or f"{dataset.view.value}.parquet")
pq.write_table(table, destination, compression="zstd")
return destination, dtypes
7 changes: 4 additions & 3 deletions src/wordstat/dtypes.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Column type inference for Wordstat's localized string values.

Wordstat exports every value as text: counts arrive as ``"5 228 679"`` with
non-breaking thousands separators, and dynamics periods as ``"01.2024"``.
non-breaking thousands separators, and dynamics periods as ``"август 2024"``
or dates such as ``"22.06.2026"``.
Writing those straight to Parquet would only change the container, so columns
are typed here first.

Expand All @@ -24,8 +25,8 @@
_WHITESPACE = re.compile(r"\s")

# Deliberately strict. ``int()``/``float()`` would accept values that are not
# numbers in this data: "01.2024" (a dynamics period) parses as 1.2024,
# silently destroying it, and "1e5", "1_000", "inf" and "nan" are accepted too.
# numbers in this data and could silently destroy a dotted date. Real monthly
# periods are "август 2024"; daily/weekly exports use "22.06.2026".
# Only a comma is a decimal separator here — Wordstat exports with a Russian
# locale — which is precisely what rejects the dotted period format.
_NUMERIC = re.compile(r"^-?[0-9]+(?:,[0-9]+)?$")
Expand Down
4 changes: 4 additions & 0 deletions src/wordstat/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ class InvalidRequestError(WordstatError):
"""The requested phrase or region is unusable before the browser is touched."""


class InvalidPeriodError(InvalidRequestError):
"""The requested dynamics granularity/window is not supported."""


class AuthenticationRequiredError(WordstatError):
"""Wordstat is reachable but the attached browser is not authenticated."""

Expand Down
5 changes: 5 additions & 0 deletions src/wordstat/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

from pydantic import BaseModel, ConfigDict, Field, computed_field, model_validator

from wordstat.periods import Granularity


class WordstatView(StrEnum):
"""The four Wordstat reports exported by the MVP."""
Expand Down Expand Up @@ -103,6 +105,9 @@ class CollectionManifest(BaseModel):
updated_at: datetime | None = None
source_url: str
exports: list[ExportSummary]
granularity: Granularity = Granularity.MONTHLY
requested_period: dict[str, str] | None = None
actual_period: dict[str, str] | None = None

@computed_field # type: ignore[prop-decorator]
@property
Expand Down
85 changes: 85 additions & 0 deletions src/wordstat/periods.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
"""Validation and normalization of user-requested dynamics periods."""

from __future__ import annotations

from calendar import monthrange
from datetime import date, timedelta
from enum import StrEnum

from wordstat.errors import InvalidPeriodError


class Granularity(StrEnum):
MONTHLY = "monthly"
WEEKLY = "weekly"
DAILY = "daily"


EARLIEST_DATE = date(2018, 1, 1)


def validate_period(
granularity: Granularity,
date_from: date | None,
date_to: date | None,
*,
today: date | None = None,
) -> None:
"""Validate an explicit window before opening Chrome.

Omitted dates leave the UI's default window untouched. The five-year
maximum is deliberately not enforced because phase 1 did not establish
it as a reliable live fact — unlike the daily lower bound below, which
is a confirmed live limitation of the picker itself, not a policy
choice.
"""
if (date_from is None) != (date_to is None):
raise InvalidPeriodError("--date-from and --date-to must be provided together")
if date_from is None or date_to is None:
return
if date_from < EARLIEST_DATE or date_to < EARLIEST_DATE:
raise InvalidPeriodError("The requested period cannot include dates before January 2018")
if date_to < date_from:
raise InvalidPeriodError("The period end date must not be earlier than its start date")
if granularity is Granularity.DAILY:
current = today or date.today()
if date_to > current:
raise InvalidPeriodError("Daily statistics cannot be requested after today")
if date_to - date_from >= timedelta(days=60):
raise InvalidPeriodError("Daily statistics support at most 60 calendar days")
# Live CDP measurement (issue #6 phase 2): the daily date-range
# picker's year-select popup only ever offers the current year, so
# a date_from further back than the trailing 60-day window cannot
# actually be selected in the live UI at all — it currently reaches
# Chrome and fails deep inside calendar-click code with an opaque
# InterfaceChangedError instead of a clear pre-flight rejection.
# Expressed relative to `today` (not "reject any year but the
# current one" — that would wrongly reject legal windows in
# January) and kept consistent with the existing 60-day window
# check above: PR #20's live-confirmed window 22.06.2026-20.08.2026,
# measured on 21.08.2026, is exactly 60 days back from today and
# must remain accepted.
if current - date_from > timedelta(days=60):
raise InvalidPeriodError("Daily statistics cannot start more than 60 days in the past")
elif granularity is Granularity.WEEKLY:
if date_to - date_from < timedelta(days=20):
raise InvalidPeriodError("Weekly statistics require at least three calendar weeks")
else:
end_month = date_from.month + 2
end_year = date_from.year + (end_month - 1) // 12
end_month = (end_month - 1) % 12 + 1
minimum_end = date(end_year, end_month, monthrange(end_year, end_month)[1])
if date_to < minimum_end:
raise InvalidPeriodError("Monthly statistics require at least three calendar months")


def parse_date(value: str | None) -> date | None:
if value is None:
return None
try:
year, month, day = (int(part) for part in value.split("-"))
if month < 1 or month > 12 or day < 1 or day > monthrange(year, month)[1]:
raise ValueError
return date(year, month, day)
except (TypeError, ValueError):
raise InvalidPeriodError(f"Invalid date {value!r}; expected YYYY-MM-DD") from None
12 changes: 12 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,18 @@ def test_phrases_file_falls_back_to_cp1251(tmp_path: Path):
assert resolve_phrases((), phrases_file) == ["чай", "кофе"]


def test_phrases_file_with_a_utf8_bom_does_not_leak_into_the_first_phrase(tmp_path: Path):
# A BOM'd UTF-8 file decodes successfully under plain "utf-8" with the
# BOM character left attached to the first line; ''.isspace() is
# False, so neither resolve_phrases' line.strip() nor the collector's
# own phrase.strip() removes it. utf-8-sig must be tried before plain
# utf-8 so the BOM is stripped during decoding itself.
phrases_file = tmp_path / "phrases.txt"
phrases_file.write_bytes("ремонт квартир\nдизайн интерьера\n".encode("utf-8-sig"))

assert resolve_phrases((), phrases_file) == ["ремонт квартир", "дизайн интерьера"]


def test_phrases_file_with_undecodable_bytes_is_reported_without_a_traceback(tmp_path: Path):
# 0x98 is invalid in both utf-8 (a lone continuation byte) and cp1251
# (unassigned in that codepage) — one of the few byte values neither
Expand Down
Loading
Loading