Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 123 additions & 22 deletions src/basic_memory/markdown/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from basic_memory.utils import normalize_project_reference
from markdown_it import MarkdownIt
from markdown_it.rules_inline.backticks import backtick
from markdown_it.token import Token

# Transcript timecodes like [00:00:11] or [1:02:03.500] share the bracket shape of
Expand Down Expand Up @@ -127,6 +128,84 @@ def parse_observation(token: Token) -> Dict[str, Any]:


# Relation handling functions
def _relation_content(token: Token) -> str:
"""Return the source content of an inline token."""
return token.tag or token.content


_CODE_SPANS_KEY = "basic_memory_code_spans"
_CODE_SPAN_PARSER = MarkdownIt()


def _record_inline_code_span(state: Any, silent: bool) -> bool:
"""Delegate to MarkdownIt and retain exact source spans for code tokens."""
start = state.pos
token_count = len(state.tokens)
matched = backtick(state, silent)
if matched and not silent and len(state.tokens) > token_count:
if state.tokens[-1].type == "code_inline":
state.env[_CODE_SPANS_KEY].append((start, state.pos))
return matched


_CODE_SPAN_PARSER.inline.ruler.at("backticks", _record_inline_code_span)


def _inline_code_spans(content: str) -> list[tuple[int, int]]:
"""Return source ranges that MarkdownIt classifies as inline code.

This delegates delimiter and escape handling to the same MarkdownIt inline
rule used by the document parser instead of duplicating CommonMark's
backtick scanner. It also preserves MarkdownIt's linear-time cache for
unmatched delimiter runs.
"""
if "`" not in content:
return []

spans: list[tuple[int, int]] = []
_CODE_SPAN_PARSER.inline.parse(content, _CODE_SPAN_PARSER, {_CODE_SPANS_KEY: spans}, [])
return spans


def _mask_wikilinks_in_inline_code(content: str, code_spans: list[tuple[int, int]]) -> str:
"""Mask only brackets in code spans, retaining all non-link source text."""
if not code_spans:
return content

masked = list(content)
for start, end in code_spans:
for position in range(start, end):
if masked[position] in "[]":
masked[position] = " "
Comment on lines +178 to +179

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve code-span brackets inside outer wikilink targets

When a real outer wikilink contains bracket text quoted by an inline code span, such as [[API [v2]]] or [[Outer [[literal]] Target]], this loop replaces those literal brackets with spaces and parse_inline_relations then extracts the target from that masked string. The source-preserving extraction added for explicit relations does not cover this implicit path, so every parse or reindex persistently creates an edge to the wrong target; scan using the masked content but extract the target from the original source positions.

AGENTS.md reference: AGENTS.md:L161-L165

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed as valid against the current branch, so I am not rejecting it. Addressed in 7caf720 as part of the source-preservation invariant: masked text is now used only to locate valid wikilink boundaries, while parse_inline_relations extracts the target from the same positions in original source text. Added regressions for [[API [v2]]] and [[Outer [[literal]] Target]]; both now retain their exact targets. Validation: 95 Markdown tests passed; ty, Ruff, format, and diff checks passed.


return "".join(masked)


def _remove_links_to_directive_outside_inline_code(
content: str, code_spans: list[tuple[int, int]]
) -> tuple[str, bool]:
"""Remove a terminal directive only when MarkdownIt parsed it as non-code."""
directive_content = list(content)
for start, end in code_spans:
directive_content[start:end] = " " * (end - start)

match = _LINKS_TO_DIRECTIVE.search("".join(directive_content))
if not match:
return content, False
return content[: match.start()].rstrip(), True


def _relation_parsing_content(token: Token) -> tuple[str, str, list[tuple[int, int]]]:
"""Build source-preserving relation input from MarkdownIt's code spans."""
source_content = _relation_content(token)
code_spans = _inline_code_spans(source_content)
return (
source_content,
_mask_wikilinks_in_inline_code(source_content, code_spans),
code_spans,
)


def parse_relation_type(content: str) -> str | None:
"""Return the explicit relation label before the first wikilink, if any."""
before_link = content.partition("[[")[0].strip()
Expand Down Expand Up @@ -154,15 +233,17 @@ def is_explicit_relation(token: Token) -> bool:
if token.type != "inline": # pragma: no cover
return False

# Use token.tag which contains the actual content for test tokens, fallback to content
content = (token.tag or token.content).strip()
_, content, _ = _relation_parsing_content(token)
if "[[" not in content or "]]" not in content:
return False
return _parse_explicit_relation(content) is not None


def _parse_explicit_relation(content: str) -> Dict[str, Any] | None:
def _parse_explicit_relation(
content: str, source_content: str | None = None
) -> Dict[str, Any] | None:
"""Parse ``type [[target]] (context)``, rejecting lines with a prose tail."""
source_content = source_content or content
rel_type = parse_relation_type(content)
if rel_type is None:
return None
Expand All @@ -172,7 +253,7 @@ def _parse_explicit_relation(content: str) -> Dict[str, Any] | None:
if start == -1 or end == -1:
return None

target = normalize_project_reference(content[start + 2 : end].strip())
target = normalize_project_reference(source_content[start + 2 : end].strip())
if not target:
return None

Expand All @@ -189,7 +270,8 @@ def _parse_explicit_relation(content: str) -> Dict[str, Any] | None:
if after:
if not _is_single_parenthesized(after):
return None
context = after[1:-1].strip() or None
source_after = source_content[end + 2 :].strip()
context = source_after[1:-1].strip() or None

return {"type": rel_type, "target": target, "context": context}

Expand Down Expand Up @@ -217,13 +299,23 @@ def _is_single_parenthesized(text: str) -> bool:

def parse_relation(token: Token) -> Dict[str, Any] | None:
"""Extract relation parts from token."""
# Use token.tag which contains the actual content for test tokens, fallback to content
content = (token.tag or token.content).strip()
return _parse_explicit_relation(content)
source_content, content, _ = _relation_parsing_content(token)
return _parse_explicit_relation(content, source_content)


def _is_escaped(content: str, position: int) -> bool:
"""Whether the character at ``position`` is escaped by an odd slash run."""
slash_count = 0
position -= 1
while position >= 0 and content[position] == "\\":
slash_count += 1
position -= 1
return slash_count % 2 == 1

def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
"""Find wiki-style links in regular content."""

def parse_inline_relations(content: str, source_content: str | None = None) -> List[Dict[str, Any]]:
"""Find wiki-style links, extracting targets from the original source."""
source_content = content if source_content is None else source_content
relations = []
start = 0

Expand All @@ -232,17 +324,20 @@ def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
start = content.find("[[", start)
if start == -1: # pragma: no cover
break
if _is_escaped(content, start):
start += 2
continue

# Find matching ]]
depth = 1
pos = start + 2
end = -1

while pos < len(content):
if content[pos : pos + 2] == "[[":
if content[pos : pos + 2] == "[[" and not _is_escaped(content, pos):
depth += 1
pos += 2
elif content[pos : pos + 2] == "]]":
elif content[pos : pos + 2] == "]]" and not _is_escaped(content, pos):
depth -= 1
if depth == 0:
end = pos
Expand All @@ -255,7 +350,7 @@ def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
# No matching ]] found
break

target = normalize_project_reference(content[start + 2 : end].strip())
target = normalize_project_reference(source_content[start + 2 : end].strip())
if target:
relations.append({"type": "links_to", "target": target, "context": None})

Expand Down Expand Up @@ -336,21 +431,27 @@ def relation_rule(state: Any) -> None:

# Only process inline tokens
if token.type == "inline":
content = token.tag or token.content
content_without_directive, has_directive = remove_links_to_directive(content)
source_content = _relation_content(token)
if "[[" not in source_content:
continue

code_spans = _inline_code_spans(source_content)
relation_content = _mask_wikilinks_in_inline_code(source_content, code_spans)
content_without_directive, has_directive = (
_remove_links_to_directive_outside_inline_code(relation_content, code_spans)
)

# Check for explicit relations in list items
if in_list_item and not has_directive and is_explicit_relation(token):
rel = parse_relation(token)
if in_list_item and not has_directive:
rel = _parse_explicit_relation(relation_content, source_content)
if rel:
token.meta["relations"] = [rel]
continue

# Always check for inline links in any text
else:
if "[[" in content:
rels = parse_inline_relations(content_without_directive)
if rels:
token.meta["relations"] = token.meta.get("relations", []) + rels
rels = parse_inline_relations(content_without_directive, source_content)
if rels:
token.meta["relations"] = token.meta.get("relations", []) + rels

# Add the rule after inline processing
md.core.ruler.after("inline", "relations", relation_rule)
64 changes: 64 additions & 0 deletions tests/markdown/test_relation_edge_cases.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,70 @@ def test_inline_relations():
assert len(token.meta["relations"]) == 3


def test_inline_code_wikilinks_do_not_create_relations():
"""CommonMark code spans quote wikilinks instead of linking notes (#1332)."""
code = "`"
cases = [
(
f"The template is {code}[[Inline Target]]{code}; prose [[Real Target]].",
["Real Target"],
),
(
f"{code}{code}literal {code} and [[Double Backtick Target]]{code}{code}",
[],
),
]

for source, expected_targets in cases:
parsed = parse(source)
assert [relation.target for relation in parsed.relations] == expected_targets


def test_inline_code_mask_keeps_source_outside_code_spans():
"""Masking code must not normalize escapes or discard relation context (#1332)."""
escaped = parse(r"\[[Literal]]")
assert escaped.relations == []

escaped_backticks = parse(r"\`see [[Target]]\`")
assert [relation.target for relation in escaped_backticks.relations] == ["Target"]

md = MarkdownIt().use(relation_plugin)
tokens = md.parse("- implemented_by [[Parser]] (`parse()`)")
token = next(t for t in tokens if t.type == "inline")
assert parse_relation(token) == {
"type": "implemented_by",
"target": "Parser",
"context": "`parse()`",
}

tokens = md.parse("- `example` implemented_by [[Parser]]")
token = next(t for t in tokens if t.type == "inline")
assert parse_relation(token) is None
assert token.meta["relations"] == [{"type": "links_to", "target": "Parser", "context": None}]

tokens = md.parse("- calls [[Parser]] `parse()`")
token = next(t for t in tokens if t.type == "inline")
assert parse_relation(token) is None
assert token.meta["relations"] == [{"type": "links_to", "target": "Parser", "context": None}]

literal_directive = parse("`#bm:links_to` [[Target]]")
assert [relation.target for relation in literal_directive.relations] == ["Target"]

for source, target in [
("[[API `[v2]`]]", "API `[v2]`"),
("[[Outer `[[literal]]` Target]]", "Outer `[[literal]]` Target"),
]:
parsed = parse(source)
assert [relation.target for relation in parsed.relations] == [target]


def test_many_unmatched_backtick_runs_keep_a_real_wikilink():
"""MarkdownIt's cached scanner avoids quadratic work on unmatched runs (#1332)."""
source = " ".join("`" * width for width in range(1, 801)) + " [[Target]]"
parsed = parse(source)
assert [relation.target for relation in parsed.relations] == ["Target"]


def test_prose_tail_falls_back_to_inline_link():
"""A sentence containing a wikilink must not mint a typed relation (#1260).

Expand Down