Restore bibbot arXiv detection with abstract-page metadata fallback

bin/issue_to_bibtex.py

@@ -11,7 +11,7 @@
 from bibtexparser.bibdatabase import BibDatabase
 from bibtexparser.bwriter import BibTexWriter
 
-from utils import fixBadBibFormat
+from utils import fixBadBibFormat, getArxivInfoFromPage
 
 try:
 
@@ -38,17 +38,28 @@
 
             arxiv_id = url.split('/')[-1]
             found, items = get_arxiv_info(arxiv_id, field='id')
+            if not found and re.search(r'v\d+$', arxiv_id) is not None:
+                found, items = get_arxiv_info(re.sub(r'v\d+$', '', arxiv_id), field='id')
+            if not found:
+                try:
+                    item = getArxivInfoFromPage(url, arxiv_id)
+                except requests.RequestException:
+                    item = None
+                if item is not None:
+                    found = True
+                    items = [item]
             if not found:
                 print(f"I could not find arXiv infos for {url}, maybe try without the version suffix 'v[.]'")
                 continue
             if len(items) > 1:
                 print(f'I got more than one item back from arXiv for {url}. See what I got:\n' +
-                      '\n'.join([item.title for item in items]) + '\n' +
+                      '\n'.join([item['title'] for item in items]) + '\n' +
                       f'I am taking the first one only, just FYI. I hope this is the correct one..\n\n')
-            url = items[0].link
-            title = items[0].title.replace('\n', '').replace('  ', ' ')
-            year = items[0]['published'].split('-')[0]
-            authors = items[0].authors
+            item = items[0]
+            url = item['link']
+            title = item['title'].replace('\n', '').replace('  ', ' ')
+            year = item['published'].split('-')[0]
+            authors = item['authors']
             if len(authors) > 1:
                 first_author = authors[0]["name"].split(" ")
                 authors = " and ".join([author["name"] for author in authors])
@@ -78,8 +89,9 @@
                     id = id_orig + letters[i]
                     i += 1
 
-            howpublished = "arXiv:" + items[0]["id"].split('/')[-1] + \
-                           " [" + items[0]["arxiv_primary_category"]["term"] + "]"
+            howpublished = "arXiv:" + item["id"].split('/')[-1]
+            if item.get("arxiv_primary_category", {}).get("term"):
+                howpublished += " [" + item["arxiv_primary_category"]["term"] + "]"
 
             if not duplicate:
                 bib_db = BibDatabase()
@@ -92,7 +104,7 @@
                         "howpublished": howpublished,
                         "url": url,
                         "year": year,
-                        "abstract": items[0]['summary'].replace('\n', ' ').replace('  ', ' '),
+                        "abstract": item['summary'].replace('\n', ' ').replace('  ', ' '),
                     }
                 ]
             else:

bin/test_utils.py

@@ -0,0 +1,37 @@
+import unittest
+
+from utils import parseArxivPage
+
+
+class ParseArxivPageTests(unittest.TestCase):
+    def test_parse_arxiv_page_extracts_metadata(self):
+        html = """
+        <html>
+          <head>
+            <meta name="citation_title" content="A parallel-in-time paper" />
+            <meta name="citation_author" content="Jane Doe" />
+            <meta name="citation_author" content="John Smith" />
+            <meta name="citation_date" content="2026/09/30" />
+            <meta name="citation_abstract" content="  An abstract with   extra spacing. " />
+            <meta name="citation_abstract_html_url" content="https://arxiv.org/abs/2609.24434" />
+            <meta name="citation_keywords" content="Numerical Analysis (math.NA); Analysis of PDEs (math.AP)" />
+          </head>
+        </html>
+        """
+
+        entry = parseArxivPage(html, "https://arxiv.org/abs/2609.24434", "2609.24434")
+
+        self.assertEqual(entry["title"], "A parallel-in-time paper")
+        self.assertEqual(entry["published"], "2026-09-30")
+        self.assertEqual(entry["summary"], "An abstract with extra spacing.")
+        self.assertEqual(entry["link"], "https://arxiv.org/abs/2609.24434")
+        self.assertEqual(entry["id"], "https://arxiv.org/abs/2609.24434")
+        self.assertEqual(entry["arxiv_primary_category"]["term"], "math.NA")
+        self.assertEqual(
+            entry["authors"],
+            [{"name": "Jane Doe"}, {"name": "John Smith"}],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

bin/utils.py

@@ -1,3 +1,26 @@
+import html
+import re
+from html.parser import HTMLParser
+
+import requests
+
+
+class _MetaTagParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.meta = {}
+
+    def handle_starttag(self, tag, attrs):
+        if tag.lower() != "meta":
+            return
+
+        attrs = dict(attrs)
+        name = attrs.get("name")
+        content = attrs.get("content")
+        if name is not None and content is not None:
+            self.meta.setdefault(name, []).append(html.unescape(content).strip())
+
+
 def fixBadBibFormat(bibString:str)->str:
     """
     Fixes the formatting of a BibTeX entry string by ensuring all field values are enclosed in braces.
@@ -27,4 +50,49 @@ def fixBadBibFormat(bibString:str)->str:
         if len(item) == 2 and "{" not in item[1]:
             item[1] = "{"+item[1]+"}"
             fields[i+1] = "=".join(item)
-    return "@"+",".join(fields)+"}"
+    return "@"+",".join(fields)+"}"
+
+
+def parseArxivPage(html_content:str, url:str, arxiv_id:str):
+    parser = _MetaTagParser()
+    parser.feed(html_content)
+    meta = parser.meta
+
+    def first(*names):
+        for name in names:
+            values = meta.get(name, [])
+            if len(values) > 0:
+                return values[0]
+        return ""
+
+    title = re.sub(r'\s+', ' ', first("citation_title", "og:title")).strip()
+    authors = [re.sub(r'\s+', ' ', author).strip() for author in meta.get("citation_author", []) if author.strip()]
+    published = first("citation_date", "citation_publication_date", "citation_online_date").replace("/", "-")
+    summary = re.sub(r'\s+', ' ', first("citation_abstract", "description")).strip()
+    link = first("citation_abstract_html_url", "citation_public_url") or url
+
+    category = ""
+    keywords = first("citation_keywords")
+    if keywords:
+        category_match = re.search(r'\(([^()]+)\)', keywords.split(";")[0])
+        if category_match is not None:
+            category = category_match.group(1)
+
+    if not title or len(authors) == 0 or not published or not summary:
+        return None
+
+    return {
+        "title": title,
+        "authors": [{"name": author} for author in authors],
+        "published": published,
+        "summary": summary,
+        "link": link,
+        "id": f"https://arxiv.org/abs/{arxiv_id}",
+        "arxiv_primary_category": {"term": category},
+    }
+
+
+def getArxivInfoFromPage(url:str, arxiv_id:str):
+    req = requests.get(url, timeout=30)
+    req.raise_for_status()
+    return parseArxivPage(req.text, url, arxiv_id)