#!/usr/bin/env python3
"""Build a first-pass CDN migration manifest.
This script is intentionally conservative. It maps paths covered by the CDN
docs, keeps stable roots as-is, and marks unresolved buckets for review instead
of guessing. It can read a local inventory TSV generated from STARGATE and add
local URL reference data from the dependent repositories.
"""
from __future__ import annotations
import argparse
import csv
import os
import re
import sqlite3
import subprocess
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
CDN_URL_RE = re.compile(
r"(?:https?://(?:www\.)?krisyotam\.com)?(/cdn/[^\s\"'<>),\]]+)"
)
MEDIA_EXTS = {
".avif",
".gif",
".jpeg",
".jpg",
".png",
".svg",
".webp",
}
KEEP_ROOTS = {"assets", "audio", "fonts", "games", "iso", "vr"}
MOVE_OUT_ROOTS = {"archive", "git", "git-edu", "deep-research"}
DELETE_EMPTY_ROOTS = {"art", "posts", "stills"}
@dataclass(frozen=True)
class InventoryRow:
path: str
size: int
sha256: str = ""
mime: str = ""
width: str = ""
height: str = ""
def slug_stem(path: str) -> str:
return Path(path).stem.lower().replace("_", "-").replace(" ", "-")
def canonical_name(path: str, kind: str) -> str:
ext = Path(path).suffix.lower() or ".jpg"
if kind == "person":
return f"portrait{ext}"
if kind == "platform":
return f"profile{ext}"
return f"cover{ext}"
def media_dir(medium: str, path: str) -> str:
return f"pics/media/{medium}/{slug_stem(path)}/{canonical_name(path, 'media')}"
def people_dir(role: str, path: str) -> str:
return f"pics/people/{role}/{slug_stem(path)}/{canonical_name(path, 'person')}"
def collection_path(collection: str, path: str, prefix: str) -> str:
rel = path.removeprefix(prefix).lstrip("/")
return f"pics/collections/{collection}/{rel}"
def map_path(old_path: str) -> tuple[str, str, str]:
parts = old_path.split("/")
root = parts[0]
ext = Path(old_path).suffix.lower()
if root in KEEP_ROOTS:
return old_path, "keep", "stable-root"
if root in MOVE_OUT_ROOTS:
return "", "move-out", f"{root}-is-not-cdn-content"
if root in DELETE_EMPTY_ROOTS:
return "", "review", f"{root}-expected-empty-or-retired"
if root == "video":
return "vids/" + "/".join(parts[1:]), "rename", "video-to-vids"
if root == "mp3s":
return "audio/" + "/".join(parts[1:]), "review", "mp3s-needs-audio-subdir"
if root == "wallpapers":
return "pics/collections/wallpapers/" + "/".join(parts[1:]), "move", "wallpapers-collection"
if root == "cover":
return "assets/covers/" + "/".join(parts[1:]), "review", "cover-root-likely-assets-covers"
if root == "photos":
return map_photo(old_path, parts, ext)
if root == "images":
return map_image(old_path, parts, ext)
return "", "review", "unknown-root"
def map_photo(old_path: str, parts: list[str], ext: str) -> tuple[str, str, str]:
if len(parts) < 2:
return "", "review", "photos-root-file"
bucket = parts[1]
rest = "/".join(parts[2:])
if bucket == "artists":
return people_dir("musicians", old_path), "candidate", "photos-artists-to-musicians"
if bucket == "bookstores":
return collection_path("bookstores", old_path, "photos/bookstores"), "move", "bookstores-collection"
if bucket == "characters":
return people_dir("characters", old_path), "candidate", "photos-characters-to-people-characters"
if bucket == "covers" and len(parts) >= 4:
medium = normalize_medium(parts[2])
return media_dir(medium, old_path), "candidate", "photos-covers-to-media"
if bucket == "icebergs":
return collection_path("icebergs", old_path, "photos/icebergs"), "move", "icebergs-collection"
if bucket == "logos":
return collection_path("logos", old_path, "photos/logos"), "move", "logos-collection"
if bucket == "mal":
user = parts[2] if len(parts) > 2 else slug_stem(old_path)
return f"pics/platform/mal/{user}/{canonical_name(old_path, 'platform')}", "candidate", "mal-profile"
if bucket == "notebooks":
return collection_path("krisyotam/notebooks", old_path, "photos/notebooks"), "move", "krisyotam-notebooks"
if bucket == "people" and len(parts) >= 4:
return people_dir(parts[2], old_path), "candidate", "photos-people-role"
if bucket in {"PFPs", "pfps"}:
return collection_path("pfps", old_path, f"photos/{bucket}"), "move", "pfps-collection"
if bucket in {"Misc", "blinkies", "discord-emojis"}:
collection = {
"Misc": "misc",
"blinkies": "blinkies",
"discord-emojis": "discord-emojis",
}[bucket]
return collection_path(collection, old_path, f"photos/{bucket}"), "move", "misc-collection"
if bucket in {"content", "reference", "research", "shop"}:
return collection_path("krisyotam/" + bucket, old_path, f"photos/{bucket}"), "move", "krisyotam-collection"
return f"pics/collections/{bucket}/{rest}", "review", "photos-unmapped-bucket"
def map_image(old_path: str, parts: list[str], ext: str) -> tuple[str, str, str]:
if len(parts) < 2:
return "", "review", "images-root-file"
bucket = parts[1]
direct_media = {
"anime": "anime",
"ballet": "ballet",
"console": "console",
"courses": "courses",
"film": "film",
"games": "games",
"lectures": "lectures",
"literature": "books",
"manga": "manga",
"plays": "plays",
"playlist": "albums",
"tv": "tv",
"workbooks": "books",
}
people_roles = {
"actors",
"artists",
"authors",
"ballerinas",
"characters",
"designers",
"mathematicians",
"musicians",
"philosophers",
"poets",
}
if bucket == "people" and len(parts) >= 4:
role = "characters" if parts[2] in {"anime", "film", "tv"} else parts[2]
return people_dir(role, old_path), "candidate", "images-people-role"
if bucket == "artists":
return people_dir("musicians", old_path), "candidate", "images-artists-to-musicians"
if bucket in people_roles:
return people_dir(bucket, old_path), "candidate", "images-role-at-root"
if bucket in direct_media:
return media_dir(direct_media[bucket], old_path), "candidate", f"images-{bucket}-to-media"
if bucket == "media":
return "", "review", "images-media-needs-medium-sort"
if bucket == "covers" and len(parts) >= 4:
return media_dir(normalize_medium(parts[2]), old_path), "candidate", "images-covers-to-media"
if bucket == "anime-girls-with-programming-books":
return collection_path("anime-girls-with-programming-books", old_path, "images/anime-girls-with-programming-books"), "move", "programming-books-collection"
if bucket in {"blinkies", "discord-emojis", "bookstores", "icebergs"}:
return collection_path(bucket, old_path, f"images/{bucket}"), "move", f"{bucket}-collection"
if bucket in {"art", "reference", "research", "shop"}:
return collection_path(bucket, old_path, f"images/{bucket}"), "move", f"{bucket}-collection"
if bucket in {"logos", "notebooks"}:
return collection_path("krisyotam/" + bucket, old_path, f"images/{bucket}"), "review", f"images-{bucket}-needs-subsort"
if bucket == "mal":
user = parts[2] if len(parts) > 2 else slug_stem(old_path)
return f"pics/platform/mal/{user}/{canonical_name(old_path, 'platform')}", "candidate", "mal-profile"
if bucket in {"pfps", "PFPs"}:
return collection_path("pfps", old_path, f"images/{bucket}"), "move", "pfps-collection"
if bucket in {"content", "posts"}:
return collection_path("krisyotam/content", old_path, f"images/{bucket}"), "review", "content-posts-needs-route-review"
return "", "review", "images-unmapped-bucket"
def normalize_medium(value: str) -> str:
table = {
"literature": "books",
"workbooks": "books",
"movies": "film",
"posters": "film",
"playlist": "albums",
}
return table.get(value, value)
def load_inventory(path: Path) -> list[InventoryRow]:
rows: list[InventoryRow] = []
with path.open(newline="") as fh:
reader = csv.DictReader(fh, delimiter="\t")
for row in reader:
rows.append(
InventoryRow(
path=row["path"],
size=int(row.get("size") or 0),
sha256=row.get("sha256", ""),
mime=row.get("mime", ""),
width=row.get("width", ""),
height=row.get("height", ""),
)
)
return rows
def collect_references(paths: list[Path], photos_db: Path | None) -> dict[str, list[str]]:
refs: dict[str, list[str]] = defaultdict(list)
for base in paths:
if not base.exists():
continue
cmd = ["rg", "-n", "--no-heading", r"(/cdn/|krisyotam\.com/cdn/)", str(base)]
proc = subprocess.run(cmd, text=True, capture_output=True, check=False)
for line in proc.stdout.splitlines():
for match in CDN_URL_RE.findall(line):
cdn_path = match.removeprefix("/cdn/")
refs[cdn_path].append(line)
if photos_db and photos_db.exists():
conn = sqlite3.connect(photos_db)
try:
for (url,) in conn.execute("select url from images where url like '%/cdn/%'"):
for match in CDN_URL_RE.findall(url):
refs[match.removeprefix("/cdn/")].append(f"{photos_db}:images.url")
finally:
conn.close()
return refs
def write_manifest(rows: list[InventoryRow], refs: dict[str, list[str]], output: Path) -> None:
with output.open("w", newline="") as fh:
fieldnames = [
"old_path",
"new_path",
"old_url",
"new_url",
"size",
"sha256",
"mime",
"width",
"height",
"status",
"reason",
"reference_count",
]
writer = csv.DictWriter(fh, fieldnames=fieldnames, delimiter="\t")
writer.writeheader()
for row in rows:
new_path, status, reason = map_path(row.path)
writer.writerow(
{
"old_path": row.path,
"new_path": new_path,
"old_url": f"/cdn/{row.path}",
"new_url": f"/cdn/{new_path}" if new_path else "",
"size": row.size,
"sha256": row.sha256,
"mime": row.mime,
"width": row.width,
"height": row.height,
"status": status,
"reason": reason,
"reference_count": len(refs.get(row.path, [])),
}
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--inventory", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--krisyotam", type=Path, default=Path("/home/krisyotam/src/krisyotam.com"))
parser.add_argument("--lit", type=Path, default=Path("/home/krisyotam/lit/content"))
parser.add_argument("--photos", type=Path, default=Path("/home/krisyotam/src/photos.krisyotam.com"))
parser.add_argument("--photos-db", type=Path, default=Path("/home/krisyotam/src/photos.krisyotam.com/public/images.db"))
args = parser.parse_args()
rows = load_inventory(args.inventory)
refs = collect_references([args.krisyotam, args.lit, args.photos], args.photos_db)
args.output.parent.mkdir(parents=True, exist_ok=True)
write_manifest(rows, refs, args.output)
return 0
if __name__ == "__main__":
raise SystemExit(main())