How the auto dependency update broke our production

# scripts/check_dependency_updates.py
import concurrent.futures
import datetime
import json
import re
import sys
import tomllib
import urllib.error
import urllib.request
from dataclasses import dataclass
from pathlib import Path

PYPROJECT_PATH = Path(__file__).resolve().parent.parent / "pyproject.toml"

PRE_RELEASE_PATTERN = re.compile(r"(a|b|rc|dev|alpha|beta)\d*", re.IGNORECASE)

EXACT_PIN_PATTERN = re.compile(r"^([a-zA-Z0-9_-]+)(?:\[[^\]]+\])?==(.+)$")
RANGE_PIN_PATTERN = re.compile(r"^([a-zA-Z0-9_-]+)(?:\[[^\]]+\])?>=([^,]+),<(.+)$")

GITHUB_REPO_PATTERN = re.compile(r"github\.com/([^/]+/[^/]+?)(?:\.git|/|$)")

MAX_RELEASES_PER_PACKAGE = 10

SEMVER_PATTERN = re.compile(r"^(\d+)\.(\d+)\.(\d+)")


@dataclass
class Dependency:
    name: str
    current_version: str


@dataclass
class UpdateInfo:
    dep: Dependency
    latest_version: str | None
    is_outdated: bool
    error: str | None = None
    github_repo: str | None = None


@dataclass
class ReleaseNote:
    title: str
    body: str
    bump_type: str  # "major", "minor", "patch"


def parse_dependency(raw: str) -> Dependency | None:
    raw = raw.strip()
    if not raw:
        return None

    cleaned = raw.split("#")[0].strip()

    match = EXACT_PIN_PATTERN.match(cleaned)
    if match:
        return Dependency(name=match.group(1), current_version=match.group(2))

    match = RANGE_PIN_PATTERN.match(cleaned)
    if match:
        return Dependency(name=match.group(1), current_version=match.group(2))

    return None


def parse_all_dependencies(pyproject: dict) -> dict[str, list[Dependency]]:
    groups: dict[str, list[Dependency]] = {}

    main_deps = pyproject.get("project", {}).get("dependencies", [])
    groups["dependencies"] = [
        d for raw in main_deps if (d := parse_dependency(raw)) is not None
    ]

    for group_name, group_deps in pyproject.get("dependency-groups", {}).items():
        parsed = [
            d
            for raw in group_deps
            if isinstance(raw, str) and (d := parse_dependency(raw)) is not None
        ]
        if parsed:
            groups[group_name] = parsed

    return groups


def is_pre_release(version: str) -> bool:
    return bool(PRE_RELEASE_PATTERN.search(version))


def _extract_github_repo(pypi_data: dict) -> str | None:
    info = pypi_data.get("info", {})
    project_urls = info.get("project_urls") or {}

    for key in ("Source", "Source Code", "Repository", "GitHub", "Homepage", "Home", "Code"):
        url = project_urls.get(key, "")
        match = GITHUB_REPO_PATTERN.search(url)
        if match:
            return match.group(1)

    home_page = info.get("home_page", "") or ""
    match = GITHUB_REPO_PATTERN.search(home_page)
    if match:
        return match.group(1)

    return None


def _classify_semver_bump(*, prev_version: str, curr_version: str) -> str:
    prev = SEMVER_PATTERN.match(prev_version)
    curr = SEMVER_PATTERN.match(curr_version)
    if not prev or not curr:
        return "patch"

    if int(curr.group(1)) != int(prev.group(1)):
        return "major"
    if int(curr.group(2)) != int(prev.group(2)):
        return "minor"
    return "patch"


def _fetch_github_release_notes(
    *,
    github_repo: str,
    current_version: str,
    latest_version: str,
) -> str | None:
    url = f"https://api.github.com/repos/{github_repo}/releases?per_page=50"
    try:
        req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"})
        with urllib.request.urlopen(req, timeout=15) as resp:
            releases = json.loads(resp.read().decode())
    except Exception:
        return None

    all_notes: list[ReleaseNote] = []
    prev_version = current_version
    relevant_releases: list[dict] = []
    for release in releases:
        tag = release.get("tag_name", "")
        tag_version = tag.lstrip("v")
        if tag_version == current_version:
            break
        relevant_releases.append(release)

    for release in reversed(relevant_releases):
        tag = release.get("tag_name", "")
        tag_version = tag.lstrip("v")
        body = (release.get("body") or "").strip()
        if not body:
            prev_version = tag_version
            continue
        bump = _classify_semver_bump(prev_version=prev_version, curr_version=tag_version)
        all_notes.append(ReleaseNote(
            title=release.get("name") or tag,
            body=body,
            bump_type=bump,
        ))
        prev_version = tag_version

    if not all_notes:
        return None

    if len(all_notes) <= MAX_RELEASES_PER_PACKAGE:
        selected = all_notes
    else:
        bump_priority = {"major": 0, "minor": 1, "patch": 2}
        selected = sorted(all_notes, key=lambda n: bump_priority[n.bump_type])[:MAX_RELEASES_PER_PACKAGE]
        selected = [n for n in all_notes if n in selected]

    lines: list[str] = []
    for note in selected:
        lines.append(f"### {note.title} [{note.bump_type}]\n{note.body}")

    result = "\n\n".join(lines)
    skipped = len(all_notes) - len(selected)
    if skipped > 0:
        result += f"\n\n... and {skipped} more patch release(s) — check GitHub for full history."

    return result


def fetch_changes_for_outdated(
    grouped_results: dict[str, list[UpdateInfo]],
) -> dict[str, str]:
    outdated: dict[str, UpdateInfo] = {}
    for infos in grouped_results.values():
        for info in infos:
            if info.is_outdated and info.github_repo and info.latest_version:
                outdated.setdefault(info.dep.name.lower(), info)

    if not outdated:
        return {}

    changes: dict[str, str] = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        future_to_name = {
            executor.submit(
                _fetch_github_release_notes,
                github_repo=info.github_repo,
                current_version=info.dep.current_version,
                latest_version=info.latest_version,
            ): info.dep.name
            for info in outdated.values()
        }
        for future in concurrent.futures.as_completed(future_to_name):
            name = future_to_name[future]
            result = future.result()
            if result:
                changes[name] = result

    return changes


def fetch_pypi_info(dep: Dependency) -> UpdateInfo:
    normalized_name = dep.name.lower().replace("_", "-")
    url = f"https://pypi.org/pypi/{normalized_name}/json"

    try:
        req = urllib.request.Request(url, headers={"Accept": "application/json"})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read().decode())
    except urllib.error.HTTPError as e:
        return UpdateInfo(dep=dep, latest_version=None, is_outdated=False, error=f"HTTP {e.code}")
    except Exception as e:
        return UpdateInfo(dep=dep, latest_version=None, is_outdated=False, error=str(e))

    best_version: str | None = None
    best_upload_time: datetime.datetime | None = None

    for version, files in data.get("releases", {}).items():
        if is_pre_release(version) or not files:
            continue

        upload_times = [
            datetime.datetime.strptime(f["upload_time"], "%Y-%m-%dT%H:%M:%S")
            for f in files
            if f.get("upload_time")
        ]
        if not upload_times:
            continue

        earliest_upload = min(upload_times)

        if best_upload_time is None or earliest_upload > best_upload_time:
            best_version = version
            best_upload_time = earliest_upload

    if best_version is None:
        return UpdateInfo(dep=dep, latest_version=None, is_outdated=False, error="no suitable version found")

    github_repo = _extract_github_repo(data)

    return UpdateInfo(
        dep=dep,
        latest_version=best_version,
        is_outdated=best_version != dep.current_version,
        github_repo=github_repo,
    )


def check_updates(
    groups: dict[str, list[Dependency]],
) -> dict[str, list[UpdateInfo]]:
    all_deps: dict[str, Dependency] = {}
    for group_deps in groups.values():
        for dep in group_deps:
            all_deps.setdefault(dep.name.lower(), dep)

    results: dict[str, UpdateInfo] = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        future_to_name = {
            executor.submit(fetch_pypi_info, dep): dep.name.lower()
            for dep in all_deps.values()
        }
        for future in concurrent.futures.as_completed(future_to_name):
            results[future_to_name[future]] = future.result()

    return {
        group_name: [results[dep.name.lower()] for dep in group_deps]
        for group_name, group_deps in groups.items()
    }


def format_table(grouped_results: dict[str, list[UpdateInfo]]) -> str:
    lines: list[str] = []

    for group_name, infos in grouped_results.items():
        if not infos:
            continue

        lines.append(f"\n{'=' * 70}")
        lines.append(f"  {group_name}")
        lines.append(f"{'=' * 70}")

        name_w = max(len(i.dep.name) for i in infos)
        curr_w = max(len(i.dep.current_version) for i in infos)

        lines.append(f"  {'Package':<{name_w}}  {'Current':<{curr_w}}  {'Latest':<15}  Status")
        lines.append(f"  {'-' * 66}")

        for info in sorted(infos, key=lambda i: i.dep.name.lower()):
            latest = info.latest_version or "?"
            if info.error:
                status = f"error: {info.error}"
            elif info.is_outdated:
                status = "UPDATE AVAILABLE"
            else:
                status = "up to date"
            lines.append(f"  {info.dep.name:<{name_w}}  {info.dep.current_version:<{curr_w}}  {latest:<15}  {status}")

    return "\n".join(lines)


def main() -> int:
    if not PYPROJECT_PATH.exists():
        print(f"Error: {PYPROJECT_PATH} not found", file=sys.stderr)
        return 1

    with open(PYPROJECT_PATH, "rb") as f:
        pyproject = tomllib.load(f)

    groups = parse_all_dependencies(pyproject)
    total = sum(len(deps) for deps in groups.values())
    print(f"Checking {total} dependencies across {len(groups)} group(s)...")

    grouped_results = check_updates(groups)
    print(format_table(grouped_results))

    outdated_count = sum(
        1 for infos in grouped_results.values() for info in infos if info.is_outdated
    )
    if outdated_count:
        print(f"\n{outdated_count} package(s) have updates available.")
        print("\nFetching release notes from GitHub...")
        changes = fetch_changes_for_outdated(grouped_results)
        if changes:
            print(f"\n{'=' * 80}")
            print("  RELEASE NOTES")
            print(f"{'=' * 80}")
            for pkg_name, notes in sorted(changes.items()):
                print(f"\n{'─' * 80}")
                print(f"  {pkg_name}")
                print(f"{'─' * 80}")
                print(notes)
        else:
            print("\nNo release notes found on GitHub for outdated packages.")

    return 0


if __name__ == "__main__":
    sys.exit(main())

How to Upgrade Dependencies

Check ALL dependencies in pyproject.toml for available updates, and upgrade each one with a newer version.

Follow these steps:
1. Run `python scripts/check_dependency_updates.py` to get a full picture of what's outdated and what's current.
2. For each outdated package, check its changelog on PyPI/GitHub for breaking changes, deprecations, or behavior changes.
3. Update the version pin in pyproject.toml to the exact new version (e.g., fastapi==0.133.0). Check for duplicates across dependency groups and update all occurrences consistently.
4. Run `uv lock` to update uv.lock.
5. If there are breaking changes, apply the necessary code updates.
6. Run tests to verify nothing is broken.
7. Summarize what changed, which packages were skipped and why, and any breaking changes you handled.