#!/usr/bin/env python3 """Build a first-pass CDN migration manifest. This script is intentionally conservative. It maps paths covered by the CDN docs, keeps stable roots as-is, and marks unresolved buckets for review instead of guessing. It can read a local inventory TSV generated from STARGATE and add local URL reference data from the dependent repositories. """ from __future__ import annotations import argparse import csv import os import re import sqlite3 import subprocess from collections import defaultdict from dataclasses import dataclass from pathlib import Path CDN_URL_RE = re.compile( r"(?:https?://(?:www\.)?krisyotam\.com)?(/cdn/[^\s\"'<>),\]]+)" ) MEDIA_EXTS = { ".avif", ".gif", ".jpeg", ".jpg", ".png", ".svg", ".webp", } KEEP_ROOTS = {"assets", "audio", "fonts", "games", "iso", "vr"} MOVE_OUT_ROOTS = {"archive", "git", "git-edu", "deep-research"} DELETE_EMPTY_ROOTS = {"art", "posts", "stills"} @dataclass(frozen=True) class InventoryRow: path: str size: int sha256: str = "" mime: str = "" width: str = "" height: str = "" def slug_stem(path: str) -> str: return Path(path).stem.lower().replace("_", "-").replace(" ", "-") def canonical_name(path: str, kind: str) -> str: ext = Path(path).suffix.lower() or ".jpg" if kind == "person": return f"portrait{ext}" if kind == "platform": return f"profile{ext}" return f"cover{ext}" def media_dir(medium: str, path: str) -> str: return f"pics/media/{medium}/{slug_stem(path)}/{canonical_name(path, 'media')}" def people_dir(role: str, path: str) -> str: return f"pics/people/{role}/{slug_stem(path)}/{canonical_name(path, 'person')}" def collection_path(collection: str, path: str, prefix: str) -> str: rel = path.removeprefix(prefix).lstrip("/") return f"pics/collections/{collection}/{rel}" def map_path(old_path: str) -> tuple[str, str, str]: parts = old_path.split("/") root = parts[0] ext = Path(old_path).suffix.lower() if root in KEEP_ROOTS: return old_path, "keep", "stable-root" if root in MOVE_OUT_ROOTS: return "", "move-out", f"{root}-is-not-cdn-content" if root in DELETE_EMPTY_ROOTS: return "", "review", f"{root}-expected-empty-or-retired" if root == "video": return "vids/" + "/".join(parts[1:]), "rename", "video-to-vids" if root == "mp3s": return "audio/" + "/".join(parts[1:]), "review", "mp3s-needs-audio-subdir" if root == "wallpapers": return "pics/collections/wallpapers/" + "/".join(parts[1:]), "move", "wallpapers-collection" if root == "cover": return "assets/covers/" + "/".join(parts[1:]), "review", "cover-root-likely-assets-covers" if root == "photos": return map_photo(old_path, parts, ext) if root == "images": return map_image(old_path, parts, ext) return "", "review", "unknown-root" def map_photo(old_path: str, parts: list[str], ext: str) -> tuple[str, str, str]: if len(parts) < 2: return "", "review", "photos-root-file" bucket = parts[1] rest = "/".join(parts[2:]) if bucket == "artists": return people_dir("musicians", old_path), "candidate", "photos-artists-to-musicians" if bucket == "bookstores": return collection_path("bookstores", old_path, "photos/bookstores"), "move", "bookstores-collection" if bucket == "characters": return people_dir("characters", old_path), "candidate", "photos-characters-to-people-characters" if bucket == "covers" and len(parts) >= 4: medium = normalize_medium(parts[2]) return media_dir(medium, old_path), "candidate", "photos-covers-to-media" if bucket == "icebergs": return collection_path("icebergs", old_path, "photos/icebergs"), "move", "icebergs-collection" if bucket == "logos": return collection_path("logos", old_path, "photos/logos"), "move", "logos-collection" if bucket == "mal": user = parts[2] if len(parts) > 2 else slug_stem(old_path) return f"pics/platform/mal/{user}/{canonical_name(old_path, 'platform')}", "candidate", "mal-profile" if bucket == "notebooks": return collection_path("krisyotam/notebooks", old_path, "photos/notebooks"), "move", "krisyotam-notebooks" if bucket == "people" and len(parts) >= 4: return people_dir(parts[2], old_path), "candidate", "photos-people-role" if bucket in {"PFPs", "pfps"}: return collection_path("pfps", old_path, f"photos/{bucket}"), "move", "pfps-collection" if bucket in {"Misc", "blinkies", "discord-emojis"}: collection = { "Misc": "misc", "blinkies": "blinkies", "discord-emojis": "discord-emojis", }[bucket] return collection_path(collection, old_path, f"photos/{bucket}"), "move", "misc-collection" if bucket in {"content", "reference", "research", "shop"}: return collection_path("krisyotam/" + bucket, old_path, f"photos/{bucket}"), "move", "krisyotam-collection" return f"pics/collections/{bucket}/{rest}", "review", "photos-unmapped-bucket" def map_image(old_path: str, parts: list[str], ext: str) -> tuple[str, str, str]: if len(parts) < 2: return "", "review", "images-root-file" bucket = parts[1] direct_media = { "anime": "anime", "ballet": "ballet", "console": "console", "courses": "courses", "film": "film", "games": "games", "lectures": "lectures", "literature": "books", "manga": "manga", "plays": "plays", "playlist": "albums", "tv": "tv", "workbooks": "books", } people_roles = { "actors", "artists", "authors", "ballerinas", "characters", "designers", "mathematicians", "musicians", "philosophers", "poets", } if bucket == "people" and len(parts) >= 4: role = "characters" if parts[2] in {"anime", "film", "tv"} else parts[2] return people_dir(role, old_path), "candidate", "images-people-role" if bucket == "artists": return people_dir("musicians", old_path), "candidate", "images-artists-to-musicians" if bucket in people_roles: return people_dir(bucket, old_path), "candidate", "images-role-at-root" if bucket in direct_media: return media_dir(direct_media[bucket], old_path), "candidate", f"images-{bucket}-to-media" if bucket == "media": return "", "review", "images-media-needs-medium-sort" if bucket == "covers" and len(parts) >= 4: return media_dir(normalize_medium(parts[2]), old_path), "candidate", "images-covers-to-media" if bucket == "anime-girls-with-programming-books": return collection_path("anime-girls-with-programming-books", old_path, "images/anime-girls-with-programming-books"), "move", "programming-books-collection" if bucket in {"blinkies", "discord-emojis", "bookstores", "icebergs"}: return collection_path(bucket, old_path, f"images/{bucket}"), "move", f"{bucket}-collection" if bucket in {"art", "reference", "research", "shop"}: return collection_path(bucket, old_path, f"images/{bucket}"), "move", f"{bucket}-collection" if bucket in {"logos", "notebooks"}: return collection_path("krisyotam/" + bucket, old_path, f"images/{bucket}"), "review", f"images-{bucket}-needs-subsort" if bucket == "mal": user = parts[2] if len(parts) > 2 else slug_stem(old_path) return f"pics/platform/mal/{user}/{canonical_name(old_path, 'platform')}", "candidate", "mal-profile" if bucket in {"pfps", "PFPs"}: return collection_path("pfps", old_path, f"images/{bucket}"), "move", "pfps-collection" if bucket in {"content", "posts"}: return collection_path("krisyotam/content", old_path, f"images/{bucket}"), "review", "content-posts-needs-route-review" return "", "review", "images-unmapped-bucket" def normalize_medium(value: str) -> str: table = { "literature": "books", "workbooks": "books", "movies": "film", "posters": "film", "playlist": "albums", } return table.get(value, value) def load_inventory(path: Path) -> list[InventoryRow]: rows: list[InventoryRow] = [] with path.open(newline="") as fh: reader = csv.DictReader(fh, delimiter="\t") for row in reader: rows.append( InventoryRow( path=row["path"], size=int(row.get("size") or 0), sha256=row.get("sha256", ""), mime=row.get("mime", ""), width=row.get("width", ""), height=row.get("height", ""), ) ) return rows def collect_references(paths: list[Path], photos_db: Path | None) -> dict[str, list[str]]: refs: dict[str, list[str]] = defaultdict(list) for base in paths: if not base.exists(): continue cmd = ["rg", "-n", "--no-heading", r"(/cdn/|krisyotam\.com/cdn/)", str(base)] proc = subprocess.run(cmd, text=True, capture_output=True, check=False) for line in proc.stdout.splitlines(): for match in CDN_URL_RE.findall(line): cdn_path = match.removeprefix("/cdn/") refs[cdn_path].append(line) if photos_db and photos_db.exists(): conn = sqlite3.connect(photos_db) try: for (url,) in conn.execute("select url from images where url like '%/cdn/%'"): for match in CDN_URL_RE.findall(url): refs[match.removeprefix("/cdn/")].append(f"{photos_db}:images.url") finally: conn.close() return refs def write_manifest(rows: list[InventoryRow], refs: dict[str, list[str]], output: Path) -> None: with output.open("w", newline="") as fh: fieldnames = [ "old_path", "new_path", "old_url", "new_url", "size", "sha256", "mime", "width", "height", "status", "reason", "reference_count", ] writer = csv.DictWriter(fh, fieldnames=fieldnames, delimiter="\t") writer.writeheader() for row in rows: new_path, status, reason = map_path(row.path) writer.writerow( { "old_path": row.path, "new_path": new_path, "old_url": f"/cdn/{row.path}", "new_url": f"/cdn/{new_path}" if new_path else "", "size": row.size, "sha256": row.sha256, "mime": row.mime, "width": row.width, "height": row.height, "status": status, "reason": reason, "reference_count": len(refs.get(row.path, [])), } ) def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--inventory", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--krisyotam", type=Path, default=Path("/home/krisyotam/src/krisyotam.com")) parser.add_argument("--lit", type=Path, default=Path("/home/krisyotam/lit/content")) parser.add_argument("--photos", type=Path, default=Path("/home/krisyotam/src/photos.krisyotam.com")) parser.add_argument("--photos-db", type=Path, default=Path("/home/krisyotam/src/photos.krisyotam.com/public/images.db")) args = parser.parse_args() rows = load_inventory(args.inventory) refs = collect_references([args.krisyotam, args.lit, args.photos], args.photos_db) args.output.parent.mkdir(parents=True, exist_ok=True) write_manifest(rows, refs, args.output) return 0 if __name__ == "__main__": raise SystemExit(main())