Fixed sports scrapers for box score + recap link

src/models/game.py

@@ -18,6 +18,10 @@ class Game:
         - `box_score`       The scoring summary of the game (optional)
         - `score_breakdown` The scoring breakdown of the game (optional)
         - 'ticket_link'    The ticket link for the game (optional)
+        - 'recap_link'     The recap/details link for the game (optional)
+        - 'recap_article_title' Title from the recap/story page when scraped (optional)
+        - 'recap_published_at'  Published date/time string from the recap page (optional)
+        - 'recap_article_image' Primary image URL from the recap page (optional)
     """
 
     def __init__(
@@ -37,6 +41,10 @@ def __init__(
         team=None,
         utc_date=None,
         ticket_link=None,
+        recap_link=None,
+        recap_article_title=None,
+        recap_published_at=None,
+        recap_article_image=None,
     ):
         self.id = id if id else str(ObjectId())
         self.city = city
@@ -53,6 +61,10 @@ def __init__(
         self.team = team
         self.utc_date = utc_date
         self.ticket_link = ticket_link
+        self.recap_link = recap_link
+        self.recap_article_title = recap_article_title
+        self.recap_published_at = recap_published_at
+        self.recap_article_image = recap_article_image
 
     def to_dict(self):
         """
@@ -74,6 +86,10 @@ def to_dict(self):
             "team": self.team,
             "utc_date": self.utc_date,
             "ticket_link": self.ticket_link,
+            "recap_link": self.recap_link,
+            "recap_article_title": self.recap_article_title,
+            "recap_published_at": self.recap_published_at,
+            "recap_article_image": self.recap_article_image,
         }
 
     @staticmethod
@@ -97,4 +113,8 @@ def from_dict(data) -> None:
             team=data.get("team"),
             utc_date=data.get("utc_date"),
             ticket_link=data.get("ticket_link"),
+            recap_link=data.get("recap_link"),
+            recap_article_title=data.get("recap_article_title"),
+            recap_published_at=data.get("recap_published_at"),
+            recap_article_image=data.get("recap_article_image"),
         )

src/mutations/create_game.py

@@ -18,6 +18,10 @@ class Arguments:
         score_breakdown = String(required=False)
         utc_date = String(required=False)
         ticket_link = String(required=False)
+        recap_link = String(required=False)
+        recap_article_title = String(required=False)
+        recap_published_at = String(required=False)
+        recap_article_image = String(required=False)
 
     game = Field(lambda: GameType)
 
@@ -36,7 +40,11 @@ def mutate(
         box_score=None,
         score_breakdown=None,
         utc_date=None,
-        ticket_link=None
+        ticket_link=None,
+        recap_link=None,
+        recap_article_title=None,
+        recap_published_at=None,
+        recap_article_image=None,
     ):
         game_data = {
             "city": city,
@@ -51,7 +59,11 @@ def mutate(
             "box_score": box_score,
             "score_breakdown": score_breakdown,
             "utc_date": utc_date,
-            "ticket_link": ticket_link
+            "ticket_link": ticket_link,
+            "recap_link": recap_link,
+            "recap_article_title": recap_article_title,
+            "recap_published_at": recap_published_at,
+            "recap_article_image": recap_article_image,
         }
         new_game = GameService.create_game(game_data)
-        return CreateGame(game=new_game)
+        return CreateGame(game=new_game)

src/repositories/game_repository.py

@@ -11,6 +11,14 @@
 logger = logging.getLogger(__name__)
 
 
+def _time_for_lookup(time):
+    """Return whether a concrete time should be included in a game lookup."""
+    if time is None:
+        return False
+    value = str(time).strip()
+    return bool(value) and value not in ("TBD", "TBA")
+
+
 class GameRepository:
     @staticmethod
     def find_all(limit=100, offset=0):
@@ -103,24 +111,23 @@ def find_by_data(city, date, gender, location, opponent_id, sport, state, time):
         return Game.from_dict(game_data) if game_data else None
 
     @staticmethod
-    def find_by_key_fields(city, date, gender, location, opponent_id, sport, state):
+    def find_by_key_fields(city, date, gender, location, opponent_id, sport, state, time=None):
         """
-        Find games without time for duplicate games
+        Find a game by its key fields, including a concrete time when available.
         """
         game_collection = db["game"]
-        games = list(
-            game_collection.find(
-                {
-                    "city": city,
-                    "date": date,
-                    "gender": gender,
-                    "location": location,
-                    "opponent_id": opponent_id,
-                    "sport": sport,
-                    "state": state,
-                }
-            )
-        )
+        base = {
+            "city": city,
+            "date": date,
+            "gender": gender,
+            "location": location,
+            "opponent_id": opponent_id,
+            "sport": sport,
+            "state": state,
+        }
+        if _time_for_lookup(time):
+            base["time"] = time
+        games = list(game_collection.find(base))
 
         if not games:
             return None
@@ -131,7 +138,7 @@ def find_by_key_fields(city, date, gender, location, opponent_id, sport, state):
         return [Game.from_dict(game) for game in games]
 
     @staticmethod
-    def find_by_tournament_key_fields(city, date, gender, location, sport, state):
+    def find_by_tournament_key_fields(city, date, gender, location, sport, state, time=None):
         """
         Find tournament games by location and date (excluding opponent_id).
         This is used when we need to find a tournament game that might have a placeholder team.
@@ -145,6 +152,8 @@ def find_by_tournament_key_fields(city, date, gender, location, sport, state):
             "gender": gender,
             "sport": sport,
         }
+        if _time_for_lookup(time):
+            query["time"] = time
 
         # For city, state, and location, use flexible matching
         # This allows finding games even when TBD/TBA values change to real values

src/scrapers/game_details_scrape.py

@@ -1,6 +1,7 @@
 import re
 import requests
 from bs4 import BeautifulSoup
+from urllib.parse import urljoin
 from src.utils.constants import *
 
 def clean_name(name):
@@ -22,9 +23,55 @@ def clean_name(name):
     return cleaned
 
 def fetch_page(url):
-    response = requests.get(url)
+    response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=20)
     return BeautifulSoup(response.text, 'html.parser')
 
+
+def scrape_sidearm_story_recap(url):
+    """
+    Extract headline, published time, and primary image from a Cornell Sidearm
+    story/recap page.
+    """
+    if not url:
+        return {}
+    try:
+        response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=20)
+        if response.status_code != 200:
+            return {}
+        soup = BeautifulSoup(response.text, "html.parser")
+    except Exception:
+        return {}
+    headline = soup.select_one(SIDEARM_STORY_HEADLINE)
+    time_el = soup.select_one(SIDEARM_STORY_PUBLISHED_TIME)
+    title = headline.get_text(strip=True) if headline else None
+    if not title:
+        og = soup.find("meta", property="og:title")
+        if og and og.get("content"):
+            title = og["content"].strip()
+    published_at = None
+    if time_el:
+        published_at = time_el.get_text(strip=True)
+        if not published_at and time_el.get("datetime"):
+            published_at = time_el["datetime"].strip()
+    if not published_at:
+        pmeta = soup.find("meta", property="article:published_time")
+        if pmeta and pmeta.get("content"):
+            published_at = pmeta["content"].strip()
+    image = soup.select_one(".sidearm-story-template-media img")
+    image_src = image.get("src") if image else None
+    out = {
+        "recap_article_image": (
+            urljoin(f"{BASE_URL.rstrip('/')}/", image_src)
+            if image_src
+            else None
+        )
+    }
+    if title:
+        out["recap_article_title"] = title
+    if published_at:
+        out["recap_published_at"] = published_at
+    return out
+
 def extract_teams_and_scores(box_score_section, sport):
     score_table = box_score_section.find(TAG_TABLE, class_=CLASS_SIDEARM_TABLE)
     team_names = []
@@ -53,6 +100,33 @@ def extract_teams_and_scores(box_score_section, sport):
 
     return team_names, period_scores
 
+def softball_summary(box_score_section):
+    summary = []
+    scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY})
+    if scoring_section:
+        scoring_rows = scoring_section.find(TAG_TBODY)
+        if scoring_rows:
+            for row in scoring_rows.find_all(TAG_TR):
+                team = row.find_all(TAG_TD)[0].find(TAG_IMG)[ATTR_ALT]
+                inning = row.find_all(TAG_TD)[3].text.strip()
+                desc_cell = row.find_all(TAG_TD)[4]
+                span = desc_cell.find(TAG_SPAN)
+                if span:
+                    span.extract()
+                desc = desc_cell.get_text(strip=True)
+                cornell_score = int(row.find_all(TAG_TD)[5].get_text(strip=True) or 0)
+                opp_score = int(row.find_all(TAG_TD)[6].get_text(strip=True) or 0)
+                summary.append({
+                    'team': team,
+                    'period': inning,
+                    'description': desc,
+                    'cor_score': cornell_score,
+                    'opp_score': opp_score
+                })
+    if not summary:
+        summary = [{"message": "No scoring events in this game."}]
+    return summary
+
 def soccer_summary(box_score_section):
     summary = []
     scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY})
@@ -229,6 +303,7 @@ def baseball_summary(box_score_section):
         summary = [{"message": "No scoring events in this game."}]
     return summary
 
+
 # def basketball_summary(box_score_section):
 #     summary = []
 #     scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY})
@@ -272,23 +347,21 @@ def scrape_game(url, sport):
         'field hockey': (lambda: extract_teams_and_scores(box_score_section, 'field hockey'), field_hockey_summary),
         'lacrosse': (lambda: extract_teams_and_scores(box_score_section, 'lacrosse'), lacrosse_summary),
         'baseball': (lambda: extract_teams_and_scores(box_score_section, 'baseball'), baseball_summary),
+        'softball': (lambda: extract_teams_and_scores(box_score_section, 'softball'), softball_summary),
         'basketball': (lambda: extract_teams_and_scores(box_score_section, 'basketball'), lambda _: []),
+
     }
 
     extract_teams_func, summary_func = sport_parsers.get(sport, (None, None))
 
     if extract_teams_func and summary_func:
         team_names, scores = extract_teams_func()
         scoring_summary = summary_func(box_score_section)
-
-        for event in scoring_summary:
-            if not event.get("time") and event.get("period"):
-                event["time"] = event["period"]
 
         return {
             'teams': team_names,
             'scores': scores,
             'scoring_summary': scoring_summary or [{"message": "No scoring events in this game."}]
         }
 
-    return {"error": "Sport parser not found"}
+    return {"error": "Sport parser not found"}