How to Find Orphan Pages on Static Sites: Python Checker

1cd yoursite
2hugo --gc # builds site in /public
3python3 -m venv .venv # creates a venv (virtual environment)
4pip install --upgrade pip
5pip install beautifulsoup4
6chmod +x 'path/to/script/orphan-checker.py' # give run permission 

1source .venv/bin/activate # activate venv every time before running script
2python3 'path/to/script/orphan-checker.py' # run script 
3deactivate # close venv when complete, or close CLI window

  1#!/usr/bin/env python3
  2
  3import os
  4import sys
  5import posixpath
  6from urllib.parse import urlparse
  7from bs4 import BeautifulSoup
  8
  9# CONFIGURATION
 10PUBLIC_DIR = "public" # HTML files
 11SITE_BASE = ""  # set if HTML links are absolute or same-origin only
 12CONTENT_SELECTORS = [".content.e-content"] # my content wrapper class, yours may be different or remove to extract all links from all pages
 13ANCHOR_SELECTOR = "a[href]"
 14IGNORE_PATH_PREFIXES = ["/images/", "/css/", "/js/", "/fonts/", "/assets/"]
 15IGNORE_PATTERNS = ["/tags/", "/series/", "/pages/", "/404"]  # exclude these from orphan results, these may be different for you
 16
 17def is_asset_path(p):
 18    for pre in IGNORE_PATH_PREFIXES:
 19        if p.startswith(pre):
 20            return True
 21    return False
 22
 23def is_ignored_pattern(p):
 24    for pat in IGNORE_PATTERNS:
 25        if p.startswith(pat):
 26            return True
 27    return False
 28
 29def normalize_href(href, page_dir):
 30    if not href:
 31        return None
 32    href = href.split("#")[0].strip()
 33    if href == "" or href == "#":
 34        return None
 35    if href.startswith(("mailto:", "tel:", "javascript:")):
 36        return None
 37    parsed = urlparse(href)
 38    if parsed.scheme in ("http", "https"):
 39        if SITE_BASE:
 40            base_netloc = urlparse(SITE_BASE).netloc
 41            if parsed.netloc != base_netloc:
 42                return None
 43        path = parsed.path or "/"
 44        return posixpath.normpath(path)
 45    if href.startswith("/"):
 46        return posixpath.normpath(href)
 47    # relative path, check against page_dir
 48    base = page_dir if page_dir.endswith("/") else page_dir + "/"
 49    joined = posixpath.normpath(posixpath.join(base, href))
 50    if not joined.startswith("/"):
 51        joined = "/" + joined
 52    return joined
 53
 54def collect_py_all_pages(public_dir):
 55    pages = set()
 56    for root, dirs, files in os.walk(public_dir):
 57        for fn in files:
 58            if not fn.lower().endswith(".html"):
 59                continue
 60            full = os.path.join(root, fn)
 61            rel = os.path.relpath(full, public_dir).replace(os.path.sep, "/")
 62            if fn == "index.html":
 63                dirpath = posixpath.dirname(rel)
 64                if dirpath in ("", "."):
 65                    path = "/"
 66                else:
 67                    path = "/" + dirpath
 68            else:
 69                path = "/" + rel
 70            pages.add(posixpath.normpath(path))
 71    return pages
 72
 73def collect_links_from_content(public_dir):
 74    linked = set()
 75    for root, dirs, files in os.walk(public_dir):
 76        for fn in files:
 77            if not fn.lower().endswith(".html"):
 78                continue
 79            full = os.path.join(root, fn)
 80            with open(full, "rb") as f:
 81                raw = f.read()
 82            try:
 83                soup = BeautifulSoup(raw, "html.parser")
 84            except Exception:
 85                continue
 86            rel = os.path.relpath(full, public_dir).replace(os.path.sep, "/")
 87            if fn == "index.html":
 88                page_dir = "/" + posixpath.dirname(rel)
 89                if page_dir == "/.":
 90                    page_dir = "/"
 91            else:
 92                page_dir = "/" + rel
 93            page_dir = posixpath.normpath(page_dir)
 94
 95            # Find specific content container
 96            nodes = []
 97            for sel in CONTENT_SELECTORS:
 98                nodes.extend(soup.select(sel))
 99            if not nodes:
100                continue
101
102            for node in nodes:
103                for a in node.select(ANCHOR_SELECTOR):
104                    href = a.get("href")
105                    norm = normalize_href(href, page_dir)
106                    if not norm:
107                        continue
108                    if is_asset_path(norm):
109                        continue
110                    linked.add(posixpath.normpath(norm))
111    return linked
112
113def main():
114    public = PUBLIC_DIR
115    if not os.path.isdir(public):
116        print(f"Error: public directory '{public}' not found.", file=sys.stderr)
117        sys.exit(2)
118
119    pages = collect_py_all_pages(public)
120    print(f"Collected {len(pages)} pages from {public}")
121
122    linked = collect_links_from_content(public)
123    print(f"Collected {len(linked)} links from content selectors {CONTENT_SELECTORS}")
124
125    candidate_pages = {p for p in pages if not is_ignored_pattern(p)}
126    candidate_pages.discard("/")  # exclude site root from orphan list
127
128    orphans = sorted(p for p in candidate_pages if p not in linked)
129
130    print("\nOrphan pages (not linked from .content.e-content):")
131    for o in orphans:
132        print(o)
133
134    with open("py_all_pages.txt", "w", encoding="utf-8") as f:
135        for p in sorted(pages):
136            f.write(p + "\n")
137    with open("py_linked_from_content.txt", "w", encoding="utf-8") as f:
138        for p in sorted(linked):
139            f.write(p + "\n")
140    with open("py_orphan_pages.txt", "w", encoding="utf-8") as f:
141        for p in orphans:
142            f.write(p + "\n")
143
144    print("\nWrote py_all_pages.txt, py_linked_from_content.txt, py_orphan_pages.txt")
145
146if __name__ == "__main__":
147    main()