#!/usr/bin/env python3
"""Check a reviewed CSV of public store domains; no contact collection."""
import csv
from html.parser import HTMLParser
import json
import re
import sys
import time
from urllib.error import HTTPError
from urllib.request import HTTPRedirectHandler, Request, build_opener
from urllib.robotparser import RobotFileParser

# Replace the contact URL with your own public crawler information page.
AGENT = "StoreListResearch/1.0 (+https://example.com/crawler-info)"
TIMEOUT = 15
DELAY = 2.0
MAX_BYTES = 2_000_000

class NoRedirect(HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        return None

opener = build_opener(NoRedirect())
last_request = 0.0

def fetch(url, delay=DELAY):
    global last_request
    time.sleep(max(0, delay - (time.monotonic() - last_request)))
    try:
        with opener.open(Request(url, headers={"User-Agent": AGENT}),
                         timeout=TIMEOUT) as response:
            if response.status != 200:
                raise ValueError("non-success response")
            body = response.read(MAX_BYTES + 1)
            if len(body) > MAX_BYTES:
                raise ValueError("response too large")
            return body.decode("utf-8", errors="replace")
    finally:
        last_request = time.monotonic()

class Signals(HTMLParser):
    def __init__(self):
        super().__init__()
        self.in_script = False
        self.shopify = False
        self.woo = False

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        self.in_script = self.in_script or tag == "script"
        asset = attrs.get("src", "") if tag == "script" else (
            attrs.get("href", "") if tag == "link" else "")
        if "wp-content/plugins/woocommerce/" in asset.lower():
            self.woo = True

    def handle_endtag(self, tag):
        if tag == "script":
            self.in_script = False

    def handle_data(self, data):
        if self.in_script and re.search(r"\bShopify\.(?:shop|theme)\s*=", data):
            self.shopify = True

def inspect(domain):
    result = dict(domain=domain, reachability="unknown", platform="unknown",
                  product_count="", catalog_status="not_checked")
    # Accept bare DNS names only. Review your input; do not use untrusted lists.
    if not re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9.-]*[a-zA-Z0-9])?", domain):
        result["reachability"] = "invalid_domain"
        return result
    if "." not in domain or domain.replace(".", "").isdigit():
        result["reachability"] = "invalid_domain"
        return result
    base = "https://" + domain
    policy = RobotFileParser(base + "/robots.txt")
    try:
        robots = fetch(policy.url)
        # Fail closed on syntax RobotFileParser does not fully implement.
        for line in robots.splitlines():
            rule = line.split("#", 1)[0].strip()
            key, separator, value = rule.partition(":")
            if "<html" in rule.lower() or (
                separator and key.lower() in ("allow", "disallow")
                and ("*" in value or "$" in value)
            ):
                raise ValueError("robots policy needs manual review")
        policy.parse(robots.splitlines())
    except HTTPError as error:
        if error.code == 404:
            policy.parse([])
        else:
            result["reachability"] = "robots_unavailable"
            return result
    except (OSError, ValueError):
        result["reachability"] = "robots_unavailable"
        return result
    delay = max(DELAY, policy.crawl_delay(AGENT) or 0)
    rate = policy.request_rate(AGENT)
    if rate and rate.requests > 0:
        delay = max(delay, rate.seconds / rate.requests)
    if not policy.can_fetch(AGENT, base + "/"):
        result["reachability"] = "robots_disallowed"
        return result
    try:
        signals = Signals()
        signals.feed(fetch(base + "/", delay))
        result["reachability"] = "reachable"
        if signals.shopify:
            result["platform"] = "shopify"
        elif signals.woo:
            result["platform"] = "woocommerce"
    except (OSError, ValueError):
        result["reachability"] = "request_failed"
        return result
    if result["platform"] == "woocommerce":
        return result
    catalog = base + "/products.json?limit=50"
    if not policy.can_fetch(AGENT, catalog):
        result["catalog_status"] = "robots_disallowed"
        return result
    try:
        data = json.loads(fetch(catalog, delay))
        products = data.get("products") if isinstance(data, dict) else None
        if not isinstance(products, list) or not all(
            isinstance(p, dict) and "id" in p and "variants" in p for p in products
        ):
            raise ValueError("unexpected catalog shape")
        result["product_count"] = len(products)
        result["catalog_status"] = "sampled"
        if result["platform"] == "unknown" and products:
            result["platform"] = "shopify_candidate"
    except (OSError, ValueError):
        result["catalog_status"] = "unavailable"
    return result

def main():
    if len(sys.argv) != 3 or sys.argv[1] == sys.argv[2]:
        raise SystemExit("Usage: python3 check-stores.py domains.csv checked.csv")
    with open(sys.argv[1], newline="", encoding="utf-8-sig") as source:
        reader = csv.DictReader(source)
        if "domain" not in (reader.fieldnames or []):
            raise SystemExit("Input CSV needs a domain header")
        with open(sys.argv[2], "w", newline="", encoding="utf-8") as target:
            writer = csv.DictWriter(target, fieldnames=[
                "domain", "reachability", "platform", "product_count", "catalog_status"
            ])
            writer.writeheader()
            seen = set()
            for row in reader:
                domain = (row.get("domain") or "").strip().lower().rstrip(".")
                if domain and domain not in seen:
                    seen.add(domain)
                    writer.writerow(inspect(domain))

if __name__ == "__main__":
    main()