/* See LICENSE file for copyright and license details. */ #include #include #include #include #include #include "config.h" #include "fetch.h" #include "util.h" static CURL *curl_handle = NULL; static size_t write_cb(void *contents, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; Response *resp = (Response *)userp; char *ptr; ptr = xrealloc(resp->data, resp->size + realsize + 1); resp->data = ptr; memcpy(&(resp->data[resp->size]), contents, realsize); resp->size += realsize; resp->data[resp->size] = '\0'; return realsize; } /* * Check if an HTTP status code is transient (worth retrying). * 429 = rate limited, 5xx = server errors */ static int is_transient(long code) { return code == 429 || code == 500 || code == 502 || code == 503 || code == 504; } void fetch_init(void) { curl_global_init(CURL_GLOBAL_ALL); curl_handle = curl_easy_init(); if (!curl_handle) die("curl_easy_init failed"); } void fetch_cleanup(void) { if (curl_handle) { curl_easy_cleanup(curl_handle); curl_handle = NULL; } curl_global_cleanup(); } Response * fetch_url(const char *url) { Response *resp; CURLcode res; char *ct, *effective_url; int attempt; for (attempt = 0; attempt < FETCH_MAX_RETRIES; attempt++) { if (attempt > 0) { unsigned int delay; delay = FETCH_RETRY_BASE * (1 << (attempt - 1)); warn("retry %d/%d for %s (waiting %us)", attempt, FETCH_MAX_RETRIES - 1, url, delay); sleep(delay); } resp = xmalloc(sizeof(Response)); resp->data = xmalloc(1); resp->data[0] = '\0'; resp->size = 0; resp->content_type = NULL; resp->status_code = 0; resp->final_url = NULL; curl_easy_reset(curl_handle); curl_easy_setopt(curl_handle, CURLOPT_URL, url); curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)resp); curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, USER_AGENT); curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl_handle, CURLOPT_MAXREDIRS, MAX_REDIRECTS); curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, CONNECT_TIMEOUT); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, REQUEST_TIMEOUT); curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1L); curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 2L); curl_easy_setopt(curl_handle, CURLOPT_ACCEPT_ENCODING, ""); res = curl_easy_perform(curl_handle); if (res != CURLE_OK) { /* Network-level failure */ if (res == CURLE_OPERATION_TIMEDOUT || res == CURLE_COULDNT_CONNECT || res == CURLE_GOT_NOTHING) { warn("fetch: %s: %s", url, curl_easy_strerror(res)); response_free(resp); resp = NULL; continue; } /* Non-transient curl error */ warn("fetch: %s: %s", url, curl_easy_strerror(res)); response_free(resp); return NULL; } curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &resp->status_code); ct = NULL; if (curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_TYPE, &ct) == CURLE_OK && ct) resp->content_type = xstrdup(ct); effective_url = NULL; if (curl_easy_getinfo(curl_handle, CURLINFO_EFFECTIVE_URL, &effective_url) == CURLE_OK && effective_url) resp->final_url = xstrdup(effective_url); /* Retry on transient HTTP errors */ if (is_transient(resp->status_code)) { response_free(resp); resp = NULL; continue; } return resp; } /* All retries exhausted */ warn("fetch: gave up on %s after %d attempts", url, FETCH_MAX_RETRIES); return resp; } void response_free(Response *resp) { if (!resp) return; free(resp->data); free(resp->content_type); free(resp->final_url); free(resp); }