#!/usr/bin/env python3
"""Polite cached fetcher. One request per second per host, or slower if robots Crawl-delay says so."""
import hashlib
import time
from pathlib import Path
from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser

import requests

ROOT = Path("/Users/hattr/Documents/wa-land-developers")
CACHE = ROOT / "raw" / "cache"
CACHE.mkdir(parents=True, exist_ok=True)
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/131.0.0.0 Safari/537.36"
)
SESSION = requests.Session()
SESSION.headers.update({"User-Agent": UA, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-AU,en;q=0.9"})
LAST = {}
ROBOTS = {}
# Hosts whose robots.txt we already read. Extra enforced delays.
MIN_DELAY = {
    "www.cossillwebley.com.au": 10.0,
    "cossillwebley.com.au": 10.0,
}


def cache_path(url: str) -> Path:
    h = hashlib.sha256(url.encode()).hexdigest()[:24]
    return CACHE / f"{h}.bin"


def meta_path(url: str) -> Path:
    return cache_path(url).with_suffix(".meta")


def robots_for(url: str) -> RobotFileParser:
    host = urlparse(url).netloc
    if host in ROBOTS:
        return ROBOTS[host]
    rp = RobotFileParser()
    robots_url = f"{urlparse(url).scheme}://{host}/robots.txt"
    try:
        # Do not recurse through get(); this is the robots fetch itself.
        time.sleep(max(0, 1.05 - (time.time() - LAST.get(host, 0))))
        r = SESSION.get(robots_url, timeout=30)
        LAST[host] = time.time()
        rp.parse(r.text.splitlines())
    except Exception as e:
        rp.parse(["User-agent: *", "Disallow:"])
        rp._error = str(e)  # type: ignore
    ROBOTS[host] = rp
    return rp


def allowed(url: str) -> bool:
    try:
        return robots_for(url).can_fetch(UA, url)
    except Exception:
        return True


def get(url: str, timeout=45, force=False, binary=False):
    """Return (status, final_url, content_bytes, from_cache)."""
    cp = cache_path(url)
    mp = meta_path(url)
    if cp.exists() and mp.exists() and not force:
        status, final = mp.read_text().split("\n", 1)
        return int(status), final, cp.read_bytes(), True
    if not allowed(url):
        raise PermissionError(f"robots disallow {url}")
    host = urlparse(url).netloc
    delay = MIN_DELAY.get(host, 1.05)
    wait = delay - (time.time() - LAST.get(host, 0))
    if wait > 0:
        time.sleep(wait)
    r = SESSION.get(url, timeout=timeout, allow_redirects=True)
    LAST[host] = time.time()
    cp.write_bytes(r.content)
    mp.write_text(f"{r.status_code}\n{r.url}")
    return r.status_code, r.url, r.content, False


if __name__ == "__main__":
    import sys
    url = sys.argv[1]
    status, final, content, cached = get(url)
    print(f"status={status} cached={cached} bytes={len(content)} final={final}")
    out = Path(sys.argv[2]) if len(sys.argv) > 2 else None
    if out:
        out.write_bytes(content)