/* See LICENSE file for copyright and license details. */ #include #include #include #include "detect.h" #include "util.h" /* CMS signature strings found in HTML */ static const char *wp_sigs[] = { "wp-content/", "wp-includes/", "wp-json/", "/xmlrpc.php", "name=\"generator\" content=\"WordPress", "powered by WordPress", NULL }; static const char *blogger_sigs[] = { "blogger.com", "blogspot.com", "content=\"blogger\"", "name=\"generator\" content=\"Blogger", "b:skin", "b:template", NULL }; static const char *hugo_sigs[] = { "name=\"generator\" content=\"Hugo", "powered by Hugo", "Hugo --", NULL }; static const char *jekyll_sigs[] = { "name=\"generator\" content=\"Jekyll", "powered by Jekyll", "jekyll-", NULL }; static const char *ghost_sigs[] = { "content=\"Ghost", "ghost-", "ghost/api/", "class=\"gh-", NULL }; static const char *drupal_sigs[] = { "Drupal.settings", "name=\"generator\" content=\"Drupal", "/sites/default/files/", "/modules/", NULL }; static const char *mediawiki_sigs[] = { "name=\"generator\" content=\"MediaWiki", "wgArticleId", "mw-content-text", "/wiki/", NULL }; /* Check if HTML contains any signature from a list */ static int match_sigs(const char *html, const char **sigs) { int i, hits; hits = 0; for (i = 0; sigs[i]; i++) { if (strcasestr(html, sigs[i])) hits++; } return hits; } static const char * sitetype_name(SiteType type) { switch (type) { case SITE_WORDPRESS: return "WordPress"; case SITE_BLOGGER: return "Blogger"; case SITE_HUGO: return "Hugo"; case SITE_JEKYLL: return "Jekyll"; case SITE_GHOST: return "Ghost"; case SITE_DRUPAL: return "Drupal"; case SITE_MEDIAWIKI: return "MediaWiki"; default: return "Unknown"; } } /* Extract feed URL from tags */ static char * find_feed_url(const char *html, const char *base_url) { const char *p, *href_start, *href_end; char *tag, *href; size_t tag_len, href_len; char quote; p = html; while ((p = strcasestr(p, "'); if (!end) break; tag_len = end - p; tag = xmalloc(tag_len + 1); memcpy(tag, p, tag_len); tag[tag_len] = '\0'; /* Check for RSS/Atom type */ if (strcasestr(tag, "application/rss+xml") || strcasestr(tag, "application/atom+xml")) { href_start = strcasestr(tag, "href="); if (href_start) { href_start += 5; quote = 0; if (*href_start == '"' || *href_start == '\'') quote = *href_start++; href_end = href_start; if (quote) { while (*href_end && *href_end != quote) href_end++; } else { while (*href_end && *href_end != ' ' && *href_end != '>') href_end++; } href_len = href_end - href_start; href = xmalloc(href_len + 1); memcpy(href, href_start, href_len); href[href_len] = '\0'; free(tag); /* Resolve relative URL */ if (str_starts_with(href, "http")) { return href; } else { char *resolved; resolved = url_resolve( base_url, href); free(href); return resolved; } } } free(tag); p = end + 1; } return NULL; } SiteInfo * detect_site(const char *html, const char *url) { SiteInfo *info; int wp, bl, hu, jk, gh, dr, mw; int best; char *domain; info = xmalloc(sizeof(SiteInfo)); info->type = SITE_UNKNOWN; info->name = "Unknown"; info->feed_url = NULL; info->api_url = NULL; info->sitemap_url = NULL; info->has_json_api = 0; /* Count signature matches for each CMS */ wp = match_sigs(html, wp_sigs); bl = match_sigs(html, blogger_sigs); hu = match_sigs(html, hugo_sigs); jk = match_sigs(html, jekyll_sigs); gh = match_sigs(html, ghost_sigs); dr = match_sigs(html, drupal_sigs); mw = match_sigs(html, mediawiki_sigs); /* Pick the CMS with the most signature hits */ best = 0; if (wp > best) { info->type = SITE_WORDPRESS; best = wp; } if (bl > best) { info->type = SITE_BLOGGER; best = bl; } if (hu > best) { info->type = SITE_HUGO; best = hu; } if (jk > best) { info->type = SITE_JEKYLL; best = jk; } if (gh > best) { info->type = SITE_GHOST; best = gh; } if (dr > best) { info->type = SITE_DRUPAL; best = dr; } if (mw > best) { info->type = SITE_MEDIAWIKI; best = mw; } /* Require at least 1 hit */ if (best < 1) { info->type = SITE_UNKNOWN; info->name = "Unknown"; return info; } info->name = sitetype_name(info->type); domain = url_get_domain(url); /* Set CMS-specific hints */ switch (info->type) { case SITE_WORDPRESS: info->has_json_api = 1; info->api_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/wp-json/wp/v2/") + 1); sprintf(info->api_url, "https://%s/wp-json/wp/v2/", domain); info->sitemap_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/wp-sitemap.xml") + 1); sprintf(info->sitemap_url, "https://%s/wp-sitemap.xml", domain); break; case SITE_BLOGGER: info->has_json_api = 1; /* Blogger Atom feed */ info->feed_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/feeds/posts/default") + 1); sprintf(info->feed_url, "https://%s/feeds/posts/default", domain); break; case SITE_HUGO: info->sitemap_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1); sprintf(info->sitemap_url, "https://%s/sitemap.xml", domain); break; case SITE_JEKYLL: info->sitemap_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1); sprintf(info->sitemap_url, "https://%s/sitemap.xml", domain); break; case SITE_GHOST: info->has_json_api = 1; info->api_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/ghost/api/content/") + 1); sprintf(info->api_url, "https://%s/ghost/api/content/", domain); info->sitemap_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1); sprintf(info->sitemap_url, "https://%s/sitemap.xml", domain); break; case SITE_DRUPAL: info->sitemap_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1); sprintf(info->sitemap_url, "https://%s/sitemap.xml", domain); break; case SITE_MEDIAWIKI: info->has_json_api = 1; info->api_url = xmalloc( strlen("https://") + strlen(domain) + strlen("/w/api.php") + 1); sprintf(info->api_url, "https://%s/w/api.php", domain); break; default: break; } /* Try to find feed URL from HTML if not set */ if (!info->feed_url) info->feed_url = find_feed_url(html, url); free(domain); return info; } void siteinfo_free(SiteInfo *info) { if (!info) return; free(info->feed_url); free(info->api_url); free(info->sitemap_url); free(info); } /* * Parse a simple sitemap.xml to extract URLs. * Returns array of URL strings, sets *count. * Caller frees the array and each string. */ char ** detect_sitemap_urls(SiteInfo *info, const char *domain, int *count) { char **urls; int capacity, n; (void)info; (void)domain; capacity = 64; n = 0; urls = xmalloc(capacity * sizeof(char *)); *count = n; return urls; } /* * Get additional seed URLs based on CMS type. * For WordPress: /feed/, /wp-sitemap.xml * For Hugo/Jekyll: /sitemap.xml, /index.xml * For Blogger: /feeds/posts/default */ char ** detect_seed_urls(SiteInfo *info, const char *domain, int *count) { char **urls; int n; size_t len; n = 0; urls = xmalloc(8 * sizeof(char *)); switch (info->type) { case SITE_WORDPRESS: len = strlen("https://") + strlen(domain) + strlen("/feed/") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/feed/", domain); n++; len = strlen("https://") + strlen(domain) + strlen("/wp-sitemap.xml") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/wp-sitemap.xml", domain); n++; break; case SITE_BLOGGER: len = strlen("https://") + strlen(domain) + strlen("/feeds/posts/default") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/feeds/posts/default", domain); n++; len = strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/sitemap.xml", domain); n++; break; case SITE_HUGO: /* fallthrough */ case SITE_JEKYLL: len = strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/sitemap.xml", domain); n++; len = strlen("https://") + strlen(domain) + strlen("/index.xml") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/index.xml", domain); n++; break; case SITE_GHOST: len = strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/sitemap.xml", domain); n++; break; case SITE_DRUPAL: len = strlen("https://") + strlen(domain) + strlen("/sitemap.xml") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/sitemap.xml", domain); n++; break; case SITE_MEDIAWIKI: len = strlen("https://") + strlen(domain) + strlen("/wiki/Special:AllPages") + 1; urls[n] = xmalloc(len); sprintf(urls[n], "https://%s/wiki/Special:AllPages", domain); n++; break; default: break; } *count = n; return urls; }