#!/usr/bin/env python3
"""TFLoffroad YouTube channel miner (wrapper around the shared template).

Discovers every video on https://www.youtube.com/@TFLoffroad (videos / shorts /
streams / live tabs) and downloads every available comment via
``youtube-comment-downloader`` using the supplied Netscape-format cookie file.

State, logs, and intermediate files are kept under /root/youtube/TFLoffroad so
the watchdog can resume after a crash or rest cycle.

CSV output follows RFC 4180:
  - Field separator: ``,``
  - Record separator: ``\\r\\n``  (Python csv default with ``newline=""``)
  - Quoting: QUOTE_MINIMAL — fields containing ``,``, ``"``, or CRLF are
    enclosed in ``"..."`` and any embedded ``"`` is escaped by doubling.
  - First record is the header.
These match the defaults of :mod:`csv` when files are opened with
``newline=""``, so the template's writers already comply; we re-assert the
constants here to make the contract explicit and protect against future
template edits.
"""

from __future__ import annotations

import csv
import importlib.util
import sys
from pathlib import Path


TEMPLATE = Path("/root/youtube/_template/youtube_miner.py")
BASE_DIR = Path("/root/youtube/TFLoffroad")
MODULE_NAME = "youtube_miner_template"

# Paths used by the early-exit guard below. These mirror what main() will set
# on the loaded template module — kept here so we can short-circuit before
# importing the (heavier) template module.
STATE_FILE = BASE_DIR / "crawler_state.json"
VIDEO_CSV = BASE_DIR / "TFLoffroad_youtube.csv"


def is_already_complete() -> bool:
    """True if every video is already covered by completed + failed in state."""
    try:
        import csv as _csv
        import json as _json
        if not (STATE_FILE.exists() and VIDEO_CSV.exists()):
            return False
        with VIDEO_CSV.open("r", encoding="utf-8", newline="") as fh:
            total = sum(1 for _ in _csv.DictReader(fh))
        state = _json.loads(STATE_FILE.read_text(encoding="utf-8"))
        completed = len(set(state.get("completed", [])))
        failed_n = len(state.get("failed", {}) or {})
        return total > 0 and completed + failed_n >= total
    except Exception:
        return False


def load_template():
    spec = importlib.util.spec_from_file_location(MODULE_NAME, TEMPLATE)
    if spec is None or spec.loader is None:
        raise RuntimeError(f"cannot load template crawler: {TEMPLATE}")
    module = importlib.util.module_from_spec(spec)
    # dataclass needs the module registered in sys.modules to resolve cls.__module__.
    sys.modules[MODULE_NAME] = module
    spec.loader.exec_module(module)
    return module


def main() -> int:
    if is_already_complete():
        print("crawl already complete; exiting without re-running crawler")
        return 0

    miner = load_template()

    # --- Channel configuration ---------------------------------------------
    miner.CHANNEL_HANDLE = "TFLoffroad"
    miner.CHANNEL_NAME = "TFLoffroad"
    miner.CHANNEL_BASE_URL = "https://www.youtube.com/@TFLoffroad"
    miner.CHANNEL_TABS = {
        "videos":  f"{miner.CHANNEL_BASE_URL}/videos",
        "shorts":  f"{miner.CHANNEL_BASE_URL}/shorts",
        "streams": f"{miner.CHANNEL_BASE_URL}/streams",
        "live":    f"{miner.CHANNEL_BASE_URL}/live",
    }

    # --- Filesystem layout -------------------------------------------------
    miner.BASE_DIR = BASE_DIR
    miner.VIDEO_CSV = VIDEO_CSV
    miner.COMMENTS_CSV = BASE_DIR / "TFLoffroad_youtube_comments.csv"
    miner.STATE_FILE = STATE_FILE
    miner.LOG_FILE = BASE_DIR / "crawler.log"
    miner.PID_FILE = BASE_DIR / "crawler.pid"
    miner.HEARTBEAT_FILE = BASE_DIR / "crawler.heartbeat"
    miner.LOCK_FILE = BASE_DIR / "crawler.lock"
    miner.COOKIE_FILE = BASE_DIR / "youtube_cookie.txt"
    miner.RAW_DIR = BASE_DIR / "raw_comments"
    miner.TMP_DIR = BASE_DIR / "tmp"

    # --- Throttling: 50 minutes of work, then random 3-6 minute rest -------
    miner.WORK_WINDOW = 50 * 60
    miner.REST_RANGE = (3 * 60, 6 * 60)

    # --- CSV quoting contract (RFC 4180) -----------------------------------
    # The template's csv.DictWriter calls happen inside write_videos_csv() and
    # append_comments(), which read these module-level names at call time.
    # Setting them here is enough to enforce RFC 4180 across the pipeline.
    miner.CSV_QUOTING = csv.QUOTE_MINIMAL        # RFC 4180 §2.6/2.7
    # The template's csv.DictWriter calls happen inside write_videos_csv() and
    # append_comments() with files opened using newline="" — Python's csv
    # module's default "excel" dialect already uses \r\n as the record
    # terminator (RFC 4180 §2.1), comma as the delimiter, " as the quotechar,
    # and double-quote escaping (§2.7).  Assert the dialect explicitly so any
    # future drift in the Python stdlib or accidental csv.register_dialect()
    # override would fail fast instead of producing non-conformant output.
    import io
    sample = csv.writer(io.StringIO())
    d = sample.dialect
    if d.lineterminator != "\r\n":
        raise RuntimeError(
            f"csv default dialect lineterminator is {d.lineterminator!r}, "
            "expected '\\r\\n' for RFC 4180 compliance"
        )
    if d.quotechar != '"':
        raise RuntimeError("csv default dialect quotechar must be '\"' for RFC 4180")
    if not d.doublequote:
        raise RuntimeError("csv default dialect must doublequote for RFC 4180 §2.7")
    if d.delimiter != ",":
        raise RuntimeError("csv default dialect delimiter must be ',' for RFC 4180")

    return miner.main()


if __name__ == "__main__":
    raise SystemExit(main())