1114 lines
44 KiB
Python
1114 lines
44 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import mimetypes
|
|
import posixpath
|
|
import re
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from collections.abc import Callable
|
|
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
|
from dataclasses import asdict, dataclass, field
|
|
from pathlib import Path
|
|
from typing import Literal
|
|
from urllib.parse import urljoin, urlsplit
|
|
from urllib.robotparser import RobotFileParser
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
from bs4.element import Tag
|
|
|
|
from app.config import Settings
|
|
from app.models import SUPPORTED_ASSET_TYPES
|
|
from app.pinned_transport import PinnedAddressBook, PinnedHTTPTransport
|
|
from app.url_policy import (
|
|
NetworkRules,
|
|
UrlPolicyError,
|
|
canonical_url,
|
|
hostname,
|
|
normalize_url,
|
|
resolve_and_validate,
|
|
same_site_host,
|
|
)
|
|
|
|
ResourceKind = Literal["document", "asset"]
|
|
DEFAULT_ASSET_TYPES = SUPPORTED_ASSET_TYPES
|
|
|
|
CSS_URL_RE = re.compile(r"url\(\s*(?P<quote>['\"]?)(?P<url>.*?)(?P=quote)\s*\)", re.IGNORECASE)
|
|
CSS_IMPORT_RE = re.compile(
|
|
r"@import\s+(?:url\(\s*)?(?P<quote>['\"])(?P<url>.*?)(?P=quote)\s*\)?",
|
|
re.IGNORECASE,
|
|
)
|
|
CSS_CHARSET_RE = re.compile(r"^\s*@charset\s+(['\"])[^'\"]+\1\s*;", re.IGNORECASE)
|
|
JS_DYNAMIC_IMPORT_RE = re.compile(
|
|
r"\bimport\s*\(\s*(?P<quote>['\"])(?P<url>[^'\"]+)(?P=quote)\s*\)"
|
|
)
|
|
JS_STATIC_IMPORT_RE = re.compile(
|
|
r"\b(?:import|export)\s+(?:[^'\"]*?\s+from\s+)?"
|
|
r"(?P<quote>['\"])(?P<url>[^'\"]+)(?P=quote)"
|
|
)
|
|
JS_NEW_URL_RE = re.compile(
|
|
r"\bnew\s+URL\(\s*(?P<quote>['\"])(?P<url>[^'\"]+)(?P=quote)\s*,\s*import\.meta\.url\s*\)"
|
|
)
|
|
JS_WORKER_RE = re.compile(
|
|
r"\bnew\s+(?:Worker|SharedWorker)\s*\(\s*"
|
|
r"(?P<quote>['\"])(?P<url>[^'\"]+)(?P=quote)"
|
|
)
|
|
JS_SOURCE_MAP_RE = re.compile(r"(?://[#@]\s*sourceMappingURL=)(?P<url>\S+)")
|
|
META_REFRESH_RE = re.compile(r"^(?P<delay>\s*\d+(?:\.\d+)?\s*;?\s*url\s*=\s*)(?P<url>.+)$", re.I)
|
|
MAX_RECORDED_WARNINGS = 100
|
|
MAX_RECORDED_ERRORS = 250
|
|
MAX_CONNECTION_TIMEOUT_SECONDS = 5.0
|
|
MAX_READ_TIMEOUT_SECONDS = 5.0
|
|
WINDOWS_RESERVED_NAMES = {
|
|
"CON",
|
|
"PRN",
|
|
"AUX",
|
|
"NUL",
|
|
*(f"COM{number}" for number in range(1, 10)),
|
|
*(f"LPT{number}" for number in range(1, 10)),
|
|
}
|
|
|
|
|
|
class CaptureError(RuntimeError):
|
|
"""A capture cannot produce a valid archive."""
|
|
|
|
|
|
class CaptureCancelled(CaptureError):
|
|
"""The user cancelled a capture while it was running."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CaptureOptions:
|
|
source_url: str
|
|
include_external_assets: bool
|
|
max_pages: int
|
|
max_depth: int
|
|
max_bytes: int
|
|
max_duration_seconds: int
|
|
parallel_connections: int = 6
|
|
download_asset_types: tuple[str, ...] = DEFAULT_ASSET_TYPES
|
|
skip_file_types: tuple[str, ...] = ()
|
|
use_proxy: bool = False
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CrawlRequest:
|
|
url: str
|
|
kind: ResourceKind
|
|
depth: int
|
|
referrer: str | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class FetchResult:
|
|
request: CrawlRequest
|
|
outcome: Literal["ok", "skipped", "error", "cancelled"]
|
|
final_url: str | None = None
|
|
redirects: list[str] = field(default_factory=list)
|
|
status_code: int | None = None
|
|
content_type: str = ""
|
|
encoding: str | None = None
|
|
body: bytes | None = None
|
|
message: str = ""
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ResourceRecord:
|
|
requested_url: str
|
|
final_url: str
|
|
local_path: str
|
|
kind: ResourceKind
|
|
content_type: str
|
|
status_code: int
|
|
byte_count: int
|
|
redirects: list[str]
|
|
encoding: str | None
|
|
|
|
def as_manifest(self) -> dict[str, object]:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class CrawlResult:
|
|
site_dir: Path
|
|
entry_point: str
|
|
report: dict[str, object]
|
|
stats: dict[str, int]
|
|
warnings: list[str]
|
|
|
|
|
|
class PathMapper:
|
|
"""Map origin URLs to safe, mostly human-readable archive-relative paths."""
|
|
|
|
def __init__(self, rules: NetworkRules) -> None:
|
|
self.rules = rules
|
|
self._paths: dict[str, str] = {}
|
|
self._used_path_keys: set[str] = set()
|
|
|
|
def assign(self, url: str, content_type: str) -> str:
|
|
key = canonical_url(url, self.rules)
|
|
if key in self._paths:
|
|
return self._paths[key]
|
|
|
|
parsed = urlsplit(key)
|
|
host = self._safe_segment(parsed.hostname or "unknown-host")
|
|
segments = [self._safe_segment(segment) for segment in parsed.path.split("/") if segment]
|
|
extension = self._extension_for(content_type)
|
|
|
|
if not segments or parsed.path.endswith("/"):
|
|
segments.append(f"index{extension or '.html'}")
|
|
elif content_type.startswith("text/html") and "." not in segments[-1]:
|
|
segments.append("index.html")
|
|
elif extension and "." not in segments[-1]:
|
|
segments[-1] = f"{segments[-1]}{extension}"
|
|
|
|
if parsed.query:
|
|
digest = hashlib.sha256(parsed.query.encode("utf-8")).hexdigest()[:12]
|
|
filename = segments[-1]
|
|
stem, dot, suffix = filename.rpartition(".")
|
|
segments[-1] = (
|
|
f"{stem or filename}__q-{digest}{dot}{suffix}" if dot else f"{filename}__q-{digest}"
|
|
)
|
|
|
|
candidate = "/".join([host, *segments])
|
|
if len(candidate) > 220:
|
|
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
|
|
candidate = f"{host}/_path-{digest}/{segments[-1][:80]}"
|
|
|
|
if candidate.casefold() in self._used_path_keys:
|
|
digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:12]
|
|
parent, filename = posixpath.split(candidate)
|
|
stem, dot, suffix = filename.rpartition(".")
|
|
filename = (
|
|
f"{stem or filename}__u-{digest}{dot}{suffix}" if dot else f"{filename}__u-{digest}"
|
|
)
|
|
candidate = posixpath.join(parent, filename)
|
|
|
|
self._paths[key] = candidate
|
|
self._used_path_keys.add(candidate.casefold())
|
|
return candidate
|
|
|
|
@staticmethod
|
|
def _safe_segment(value: str) -> str:
|
|
safe = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip())
|
|
safe = safe.strip(".-") or "resource"
|
|
if safe.split(".", 1)[0].upper() in WINDOWS_RESERVED_NAMES:
|
|
safe = f"_{safe}"
|
|
return safe[:80]
|
|
|
|
@staticmethod
|
|
def _extension_for(content_type: str) -> str:
|
|
media_type = content_type.split(";", 1)[0].strip().lower()
|
|
known = {
|
|
"text/html": ".html",
|
|
"text/css": ".css",
|
|
"text/javascript": ".js",
|
|
"application/javascript": ".js",
|
|
"application/json": ".json",
|
|
"application/manifest+json": ".webmanifest",
|
|
"image/svg+xml": ".svg",
|
|
"font/woff": ".woff",
|
|
"font/woff2": ".woff2",
|
|
}
|
|
if media_type in known:
|
|
return known[media_type]
|
|
return mimetypes.guess_extension(media_type) or ""
|
|
|
|
|
|
class SiteCrawler:
|
|
"""A bounded static web crawler with archive-ready link rewriting."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
settings: Settings,
|
|
options: CaptureOptions,
|
|
cancelled: Callable[[], bool],
|
|
on_progress: Callable[[str, str, dict[str, int], list[str]], None],
|
|
) -> None:
|
|
self.settings = settings
|
|
self.options = options
|
|
self.cancelled = cancelled
|
|
self.on_progress = on_progress
|
|
self.rules = NetworkRules(
|
|
allow_private_networks=settings.allow_private_networks,
|
|
allow_nonstandard_ports=settings.allow_nonstandard_ports,
|
|
)
|
|
self.source_url = normalize_url(options.source_url, self.rules)
|
|
self.source_host = hostname(self.source_url)
|
|
self.parallel_connections = max(
|
|
1, min(options.parallel_connections, self.settings.fetch_concurrency)
|
|
)
|
|
self._proxy_url = settings.proxy_url if options.use_proxy else None
|
|
self.mapper = PathMapper(self.rules)
|
|
self.pending: deque[CrawlRequest] = deque()
|
|
self.enqueued: set[str] = set()
|
|
self.url_to_path: dict[str, str] = {}
|
|
self.records: list[ResourceRecord] = []
|
|
self.errors: list[dict[str, object]] = []
|
|
self.warnings: list[str] = []
|
|
self._warning_count = 0
|
|
self._warnings_omitted = 0
|
|
self._error_count = 0
|
|
self._errors_omitted = 0
|
|
self.stats = {
|
|
"pages_found": 0,
|
|
"pages_fetched": 0,
|
|
"assets_found": 0,
|
|
"assets_fetched": 0,
|
|
"bytes_downloaded": 0,
|
|
"files_written": 0,
|
|
}
|
|
self._started_at = 0.0
|
|
self._byte_count = 0
|
|
self._byte_lock = threading.Lock()
|
|
self._scope_lock = threading.Lock()
|
|
self._robots_lock = threading.Lock()
|
|
self._external_hosts: set[str] = set()
|
|
self._robots: dict[str, RobotFileParser | bool] = {}
|
|
self._root_record: ResourceRecord | None = None
|
|
self._root_error: str | None = None
|
|
self._limit_reached = False
|
|
self._site_dir: Path | None = None
|
|
self._address_book = PinnedAddressBook()
|
|
limits = httpx.Limits(
|
|
max_connections=max(4, self.parallel_connections * 2),
|
|
max_keepalive_connections=max(2, self.parallel_connections),
|
|
)
|
|
self._client = httpx.Client(
|
|
follow_redirects=False,
|
|
trust_env=False,
|
|
timeout=httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0),
|
|
limits=limits,
|
|
transport=PinnedHTTPTransport(self._address_book, limits, proxy=self._proxy_url),
|
|
headers={"User-Agent": settings.user_agent, "Accept": "*/*"},
|
|
)
|
|
|
|
def run(self, work_dir: Path) -> CrawlResult:
|
|
self._started_at = time.monotonic()
|
|
self._site_dir = work_dir / "site"
|
|
self._site_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
if self._proxy_url:
|
|
self._pin_proxy()
|
|
self._enqueue(CrawlRequest(self.source_url, "document", 0))
|
|
executor = ThreadPoolExecutor(max_workers=self.parallel_connections)
|
|
try:
|
|
while self.pending and not self._limit_reached:
|
|
self._ensure_not_cancelled()
|
|
if self._duration_exceeded():
|
|
raise CaptureError("The configured capture time limit was reached.")
|
|
|
|
batch = [
|
|
self.pending.popleft()
|
|
for _ in range(min(len(self.pending), self.parallel_connections))
|
|
]
|
|
pending_futures = {executor.submit(self._fetch, request) for request in batch}
|
|
while pending_futures:
|
|
completed, pending_futures = wait(
|
|
pending_futures,
|
|
timeout=0.1,
|
|
return_when=FIRST_COMPLETED,
|
|
)
|
|
self._ensure_not_cancelled()
|
|
for future in completed:
|
|
self._handle_fetch_result(future.result())
|
|
|
|
self._emit_progress("Capturing", "Discovering pages and assets")
|
|
finally:
|
|
# Do not wait for a stalled remote server after a user cancellation.
|
|
executor.shutdown(wait=not self.cancelled(), cancel_futures=True)
|
|
|
|
if not self._root_record:
|
|
raise CaptureError(self._root_error or "The starting page could not be captured.")
|
|
|
|
self._ensure_within_duration()
|
|
self._emit_progress("Rewriting", "Converting captured links for offline use")
|
|
self._rewrite_captured_content()
|
|
self._ensure_within_duration()
|
|
report = self._build_report()
|
|
self._write_capture_metadata(report)
|
|
self._emit_progress("Packaging", "Preparing archive contents")
|
|
return CrawlResult(
|
|
site_dir=self._site_dir,
|
|
entry_point=self._root_record.local_path,
|
|
report=report,
|
|
stats=self.stats.copy(),
|
|
warnings=self._visible_warnings(),
|
|
)
|
|
finally:
|
|
self._client.close()
|
|
|
|
def _fetch(self, request: CrawlRequest) -> FetchResult:
|
|
current_url = request.url
|
|
redirects: list[str] = []
|
|
|
|
for _ in range(self.settings.max_redirects + 1):
|
|
if self.cancelled():
|
|
return FetchResult(request, "cancelled", message="Capture cancelled")
|
|
if self._duration_exceeded():
|
|
raise CaptureError("The configured capture time limit was reached.")
|
|
|
|
try:
|
|
self._pin_target(current_url)
|
|
except UrlPolicyError as error:
|
|
return FetchResult(request, "skipped", final_url=current_url, message=str(error))
|
|
|
|
if not self._robots_allow(current_url):
|
|
return FetchResult(
|
|
request,
|
|
"skipped",
|
|
final_url=current_url,
|
|
message="robots.txt excludes this resource",
|
|
)
|
|
|
|
try:
|
|
headers = self._referer_header(request.referrer, current_url)
|
|
with self._client.stream(
|
|
"GET",
|
|
current_url,
|
|
headers=headers,
|
|
timeout=self._request_timeout(),
|
|
) as response:
|
|
status_code = response.status_code
|
|
if status_code in {301, 302, 303, 307, 308}:
|
|
location = response.headers.get("location")
|
|
if not location:
|
|
return FetchResult(
|
|
request,
|
|
"error",
|
|
final_url=current_url,
|
|
status_code=status_code,
|
|
message="Redirect response had no Location header",
|
|
)
|
|
try:
|
|
next_url = normalize_url(urljoin(current_url, location), self.rules)
|
|
except UrlPolicyError as error:
|
|
return FetchResult(
|
|
request, "skipped", final_url=current_url, message=str(error)
|
|
)
|
|
if not self._scope_allows(next_url, request.kind):
|
|
return FetchResult(
|
|
request,
|
|
"skipped",
|
|
final_url=current_url,
|
|
message="Redirect leaves the configured capture scope",
|
|
)
|
|
redirects.append(current_url)
|
|
current_url = next_url
|
|
continue
|
|
|
|
if status_code >= 400:
|
|
return FetchResult(
|
|
request,
|
|
"error",
|
|
final_url=current_url,
|
|
status_code=status_code,
|
|
message=f"Remote server returned HTTP {status_code}",
|
|
)
|
|
|
|
body_or_error = self._read_limited_response(response)
|
|
if isinstance(body_or_error, str):
|
|
return FetchResult(
|
|
request,
|
|
"skipped",
|
|
final_url=current_url,
|
|
status_code=status_code,
|
|
message=body_or_error,
|
|
)
|
|
|
|
return FetchResult(
|
|
request,
|
|
"ok",
|
|
final_url=current_url,
|
|
redirects=redirects,
|
|
status_code=status_code,
|
|
content_type=response.headers.get("content-type", "")
|
|
.split(";", 1)[0]
|
|
.lower(),
|
|
encoding=response.encoding,
|
|
body=body_or_error,
|
|
)
|
|
except httpx.HTTPError as error:
|
|
return FetchResult(
|
|
request, "error", final_url=current_url, message=f"Network error: {error}"
|
|
)
|
|
|
|
return FetchResult(request, "skipped", final_url=current_url, message="Too many redirects")
|
|
|
|
def _read_limited_response(
|
|
self, response: httpx.Response, maximum_bytes: int | None = None
|
|
) -> bytes | str:
|
|
chunks: list[bytes] = []
|
|
response_bytes = 0
|
|
response_limit = maximum_bytes or self.settings.max_response_bytes
|
|
for chunk in response.iter_bytes():
|
|
if self.cancelled():
|
|
raise CaptureCancelled("Capture cancelled")
|
|
if self._duration_exceeded():
|
|
raise CaptureError("The configured capture time limit was reached.")
|
|
response_bytes += len(chunk)
|
|
if response_bytes > response_limit:
|
|
return "Resource exceeds the per-file size limit"
|
|
with self._byte_lock:
|
|
if self._byte_count + len(chunk) > self.options.max_bytes:
|
|
self._limit_reached = True
|
|
return "The configured total download budget was reached"
|
|
self._byte_count += len(chunk)
|
|
self.stats["bytes_downloaded"] = self._byte_count
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
|
|
def _handle_fetch_result(self, result: FetchResult) -> None:
|
|
request_key = canonical_url(result.request.url, self.rules)
|
|
is_root = request_key == canonical_url(self.source_url, self.rules)
|
|
|
|
if result.outcome == "cancelled":
|
|
raise CaptureCancelled("Capture cancelled")
|
|
if result.outcome != "ok" or not result.final_url or result.body is None:
|
|
if is_root:
|
|
self._root_error = result.message or "The starting page could not be fetched"
|
|
self._record_problem(result)
|
|
return
|
|
|
|
local_path = self.mapper.assign(result.final_url, result.content_type)
|
|
try:
|
|
destination = self._safe_destination(local_path)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(result.body)
|
|
except OSError as error:
|
|
failed_result = FetchResult(
|
|
result.request,
|
|
"error",
|
|
final_url=result.final_url,
|
|
status_code=result.status_code,
|
|
message=f"Could not write this resource locally: {error}",
|
|
)
|
|
if is_root:
|
|
self._root_error = failed_result.message
|
|
self._record_problem(failed_result)
|
|
return
|
|
|
|
record = ResourceRecord(
|
|
requested_url=result.request.url,
|
|
final_url=result.final_url,
|
|
local_path=local_path,
|
|
kind=result.request.kind,
|
|
content_type=result.content_type,
|
|
status_code=result.status_code or 200,
|
|
byte_count=len(result.body),
|
|
redirects=result.redirects,
|
|
encoding=result.encoding,
|
|
)
|
|
self.records.append(record)
|
|
self.url_to_path[request_key] = local_path
|
|
self.url_to_path[canonical_url(result.final_url, self.rules)] = local_path
|
|
for redirect in result.redirects:
|
|
self.url_to_path[canonical_url(redirect, self.rules)] = local_path
|
|
|
|
self.stats["files_written"] += 1
|
|
if result.request.kind == "document":
|
|
self.stats["pages_fetched"] += 1
|
|
else:
|
|
self.stats["assets_fetched"] += 1
|
|
|
|
if is_root:
|
|
self._root_record = record
|
|
|
|
text = self._decode_text(result.body, result.encoding)
|
|
if self._is_html(record, text):
|
|
self._discover_html(text, record.final_url, result.request.depth)
|
|
elif self._is_css(record):
|
|
self._discover_css(text, record.final_url, result.request.depth)
|
|
elif self._is_javascript(record):
|
|
self._discover_javascript(text, record.final_url, result.request.depth)
|
|
|
|
def _record_problem(self, result: FetchResult) -> None:
|
|
if result.outcome == "skipped" and result.message:
|
|
self._add_warning(f"Skipped {result.request.url}: {result.message}")
|
|
elif result.message:
|
|
self._error_count += 1
|
|
if len(self.errors) < MAX_RECORDED_ERRORS:
|
|
self.errors.append(
|
|
{
|
|
"url": result.request.url,
|
|
"kind": result.request.kind,
|
|
"status_code": result.status_code,
|
|
"message": result.message,
|
|
}
|
|
)
|
|
else:
|
|
self._errors_omitted += 1
|
|
|
|
def _discover_html(self, text: str, page_url: str, depth: int) -> None:
|
|
soup = BeautifulSoup(text, "html.parser")
|
|
base_url = page_url
|
|
base = soup.find("base", href=True)
|
|
if isinstance(base, Tag):
|
|
resolved = self._resolve_reference(page_url, self._tag_value(base, "href") or "")
|
|
if resolved:
|
|
base_url = resolved
|
|
|
|
for tag in soup.find_all(True):
|
|
if not isinstance(tag, Tag):
|
|
continue
|
|
name = (tag.name or "").lower()
|
|
document_attribute = "href" if name in {"a", "area"} else "src"
|
|
document_url = self._tag_value(tag, document_attribute)
|
|
if name in {"a", "area", "iframe", "frame"} and document_url:
|
|
attribute = "href" if name in {"a", "area"} else "src"
|
|
self._enqueue_reference(
|
|
self._tag_value(tag, attribute), base_url, "document", depth + 1, page_url
|
|
)
|
|
|
|
link_url = self._tag_value(tag, "href")
|
|
if name == "link" and link_url:
|
|
rel_value = tag.get("rel")
|
|
if isinstance(rel_value, list):
|
|
rel = {str(item).lower() for item in rel_value}
|
|
elif isinstance(rel_value, str):
|
|
rel = {rel_value.lower()}
|
|
else:
|
|
rel = set()
|
|
if not rel.intersection({"canonical", "alternate"}):
|
|
self._enqueue_reference(link_url, base_url, "asset", depth, page_url)
|
|
|
|
for attribute in ("src", "poster", "data", "background"):
|
|
attribute_url = self._tag_value(tag, attribute)
|
|
if attribute_url:
|
|
kind: ResourceKind = (
|
|
"document" if name == "object" and attribute == "data" else "asset"
|
|
)
|
|
self._enqueue_reference(attribute_url, base_url, kind, depth, page_url)
|
|
|
|
if name in {"use", "image"}:
|
|
for attribute in ("href", "xlink:href"):
|
|
attribute_url = self._tag_value(tag, attribute)
|
|
if attribute_url:
|
|
self._enqueue_reference(attribute_url, base_url, "asset", depth, page_url)
|
|
|
|
srcset = self._tag_value(tag, "srcset")
|
|
if srcset:
|
|
for candidate in self._srcset_urls(srcset):
|
|
self._enqueue_reference(candidate, base_url, "asset", depth, page_url)
|
|
|
|
style = self._tag_value(tag, "style")
|
|
if style:
|
|
self._discover_css(style, base_url, depth)
|
|
|
|
if name == "style" and tag.string:
|
|
self._discover_css(str(tag.string), base_url, depth)
|
|
if name == "script" and tag.string:
|
|
self._discover_javascript(str(tag.string), base_url, depth)
|
|
if name == "meta":
|
|
property_name = str(tag.get("property") or tag.get("name") or "").lower()
|
|
content = self._tag_value(tag, "content")
|
|
if content and property_name in {"og:image", "twitter:image", "og:video"}:
|
|
self._enqueue_reference(content, base_url, "asset", depth, page_url)
|
|
|
|
for meta in soup.find_all("meta"):
|
|
if not isinstance(meta, Tag):
|
|
continue
|
|
content = self._tag_value(meta, "content")
|
|
if str(meta.get("http-equiv", "")).lower() == "refresh" and content:
|
|
match = META_REFRESH_RE.match(content)
|
|
if match:
|
|
self._enqueue_reference(
|
|
match.group("url"), base_url, "document", depth + 1, page_url
|
|
)
|
|
|
|
def _discover_css(self, text: str, base_url: str, depth: int) -> None:
|
|
for match in CSS_URL_RE.finditer(text):
|
|
self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url)
|
|
for match in CSS_IMPORT_RE.finditer(text):
|
|
self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url)
|
|
|
|
def _discover_javascript(self, text: str, base_url: str, depth: int) -> None:
|
|
for pattern in (
|
|
JS_DYNAMIC_IMPORT_RE,
|
|
JS_STATIC_IMPORT_RE,
|
|
JS_NEW_URL_RE,
|
|
JS_WORKER_RE,
|
|
JS_SOURCE_MAP_RE,
|
|
):
|
|
for match in pattern.finditer(text):
|
|
self._enqueue_reference(match.group("url"), base_url, "asset", depth, base_url)
|
|
|
|
def _enqueue_reference(
|
|
self, raw_url: object, base_url: str, kind: ResourceKind, depth: int, referrer: str | None
|
|
) -> None:
|
|
if not isinstance(raw_url, str):
|
|
return
|
|
resolved = self._resolve_reference(base_url, raw_url)
|
|
if resolved:
|
|
self._enqueue(CrawlRequest(resolved, kind, depth, referrer))
|
|
|
|
def _enqueue(self, request: CrawlRequest) -> None:
|
|
try:
|
|
key = canonical_url(request.url, self.rules)
|
|
except UrlPolicyError:
|
|
return
|
|
if key in self.enqueued or self._limit_reached:
|
|
return
|
|
if request.kind == "document" and request.depth > self.options.max_depth:
|
|
self._add_warning(f"Skipped {request.url}: page depth limit reached")
|
|
return
|
|
if len(self.enqueued) >= self.settings.max_files:
|
|
self._add_warning("The configured file-count safety limit was reached.")
|
|
self._limit_reached = True
|
|
return
|
|
if request.kind == "document" and self.stats["pages_found"] >= self.options.max_pages:
|
|
self._add_warning("The configured page limit was reached.")
|
|
return
|
|
if not self._scope_allows(key, request.kind):
|
|
return
|
|
if request.kind == "asset" and not self._asset_is_allowed(key):
|
|
return
|
|
|
|
self.enqueued.add(key)
|
|
self.pending.append(CrawlRequest(key, request.kind, request.depth, request.referrer))
|
|
if request.kind == "document":
|
|
self.stats["pages_found"] += 1
|
|
else:
|
|
self.stats["assets_found"] += 1
|
|
|
|
def _scope_allows(self, url: str, kind: ResourceKind) -> bool:
|
|
candidate_host = hostname(url)
|
|
if same_site_host(candidate_host, self.source_host):
|
|
return True
|
|
if kind == "document":
|
|
return False
|
|
if not self.options.include_external_assets:
|
|
return False
|
|
with self._scope_lock:
|
|
if candidate_host not in self._external_hosts:
|
|
if len(self._external_hosts) >= self.settings.max_external_hosts:
|
|
self._add_warning(
|
|
"External asset host limit reached; remaining third-party assets "
|
|
"were skipped."
|
|
)
|
|
return False
|
|
self._external_hosts.add(candidate_host)
|
|
return True
|
|
|
|
def _asset_is_allowed(self, url: str) -> bool:
|
|
extension = posixpath.splitext(urlsplit(url).path.lower())[1].lstrip(".")
|
|
if extension and extension in self.options.skip_file_types:
|
|
self._add_warning(f"Skipped .{extension} files because of the configured file rule.")
|
|
return False
|
|
asset_type = self._asset_type(url)
|
|
if asset_type not in self.options.download_asset_types:
|
|
self._add_warning(
|
|
f"Skipped {asset_type} assets because they are not selected for this capture."
|
|
)
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def _asset_type(url: str) -> str:
|
|
extension = posixpath.splitext(urlsplit(url).path.lower())[1].lstrip(".")
|
|
if extension == "css":
|
|
return "styles"
|
|
if extension in {"js", "mjs", "cjs", "map"}:
|
|
return "scripts"
|
|
if extension in {"avif", "bmp", "gif", "ico", "jpeg", "jpg", "png", "svg", "webp"}:
|
|
return "images"
|
|
if extension in {"eot", "otf", "ttf", "woff", "woff2"}:
|
|
return "fonts"
|
|
if extension in {
|
|
"aac",
|
|
"avi",
|
|
"m4a",
|
|
"mkv",
|
|
"mov",
|
|
"mp3",
|
|
"mp4",
|
|
"mpeg",
|
|
"oga",
|
|
"ogg",
|
|
"ogv",
|
|
"wav",
|
|
"webm",
|
|
}:
|
|
return "media"
|
|
return "other"
|
|
|
|
def _robots_allow(self, url: str) -> bool:
|
|
if not self.settings.respect_robots:
|
|
return True
|
|
parsed = urlsplit(url)
|
|
origin = f"{parsed.scheme}://{parsed.netloc}"
|
|
with self._robots_lock:
|
|
cached = self._robots.get(origin)
|
|
known_origin = origin in self._robots
|
|
if not known_origin:
|
|
loaded = self._load_robots(origin)
|
|
with self._robots_lock:
|
|
cached = self._robots.setdefault(origin, loaded)
|
|
if isinstance(cached, RobotFileParser):
|
|
return cached.can_fetch(self.settings.user_agent, url)
|
|
return True
|
|
|
|
def _load_robots(self, origin: str) -> RobotFileParser | bool:
|
|
robots_url = f"{origin}/robots.txt"
|
|
parser = RobotFileParser()
|
|
parser.set_url(robots_url)
|
|
try:
|
|
self._pin_target(robots_url)
|
|
with self._client.stream(
|
|
"GET",
|
|
robots_url,
|
|
follow_redirects=False,
|
|
timeout=self._request_timeout(),
|
|
) as response:
|
|
if response.status_code == 200:
|
|
body = self._read_limited_response(response, maximum_bytes=512 * 1024)
|
|
if isinstance(body, bytes):
|
|
parser.parse(self._decode_text(body, response.encoding).splitlines())
|
|
return parser
|
|
return True
|
|
if response.status_code in {401, 403}:
|
|
parser.parse(["User-agent: *", "Disallow: /"])
|
|
return parser
|
|
except (httpx.HTTPError, UrlPolicyError):
|
|
return True
|
|
return True
|
|
|
|
def _rewrite_captured_content(self) -> None:
|
|
for record in self.records:
|
|
self._ensure_not_cancelled()
|
|
self._ensure_within_duration()
|
|
destination = self._safe_destination(record.local_path)
|
|
if self._is_html(record):
|
|
text = self._decode_text(destination.read_bytes(), record.encoding)
|
|
destination.write_text(self._rewrite_html(text, record), encoding="utf-8")
|
|
elif self._is_css(record):
|
|
text = self._decode_text(destination.read_bytes(), record.encoding)
|
|
rewritten = self._rewrite_css(text, record.local_path, record.final_url)
|
|
rewritten = CSS_CHARSET_RE.sub('@charset "UTF-8";', rewritten, count=1)
|
|
destination.write_text(
|
|
rewritten,
|
|
encoding="utf-8",
|
|
)
|
|
elif self._is_javascript(record):
|
|
text = self._decode_text(destination.read_bytes(), record.encoding)
|
|
destination.write_text(
|
|
self._rewrite_javascript(text, record.local_path, record.final_url),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _rewrite_html(self, text: str, record: ResourceRecord) -> str:
|
|
soup = BeautifulSoup(text, "html.parser")
|
|
base_url = record.final_url
|
|
base = soup.find("base", href=True)
|
|
if isinstance(base, Tag):
|
|
resolved = self._resolve_reference(base_url, self._tag_value(base, "href") or "")
|
|
if resolved:
|
|
base_url = resolved
|
|
base.decompose()
|
|
|
|
for meta in soup.find_all("meta", charset=True):
|
|
if isinstance(meta, Tag):
|
|
meta["charset"] = "utf-8"
|
|
for meta in soup.find_all("meta"):
|
|
if not isinstance(meta, Tag):
|
|
continue
|
|
if str(meta.get("http-equiv", "")).lower() != "content-type":
|
|
continue
|
|
content = self._tag_value(meta, "content")
|
|
if content and "charset=" in content.lower():
|
|
meta["content"] = re.sub(
|
|
r"charset\s*=\s*[^;\s]+", "charset=UTF-8", content, flags=re.I
|
|
)
|
|
|
|
for tag in soup.find_all(True):
|
|
if not isinstance(tag, Tag):
|
|
continue
|
|
for attribute in ("href", "src", "poster", "data", "background", "xlink:href"):
|
|
attribute_url = self._tag_value(tag, attribute)
|
|
if attribute_url:
|
|
tag[attribute] = self._local_reference(
|
|
record.local_path, base_url, attribute_url
|
|
)
|
|
srcset = self._tag_value(tag, "srcset")
|
|
if srcset:
|
|
tag["srcset"] = self._rewrite_srcset(record.local_path, base_url, srcset)
|
|
style = self._tag_value(tag, "style")
|
|
if style:
|
|
tag["style"] = self._rewrite_css(style, record.local_path, base_url)
|
|
name = (tag.name or "").lower()
|
|
if name == "style" and tag.string:
|
|
style_text = str(tag.string)
|
|
tag.clear()
|
|
tag.append(self._rewrite_css(style_text, record.local_path, base_url))
|
|
if name == "script" and tag.string:
|
|
script_text = str(tag.string)
|
|
tag.clear()
|
|
tag.append(self._rewrite_javascript(script_text, record.local_path, base_url))
|
|
|
|
for meta in soup.find_all("meta"):
|
|
if not isinstance(meta, Tag):
|
|
continue
|
|
content = self._tag_value(meta, "content")
|
|
if str(meta.get("http-equiv", "")).lower() == "refresh" and content:
|
|
match = META_REFRESH_RE.match(content)
|
|
if match:
|
|
meta["content"] = match.group("delay") + self._local_reference(
|
|
record.local_path, base_url, match.group("url")
|
|
)
|
|
return str(soup)
|
|
|
|
def _rewrite_css(self, text: str, local_path: str, base_url: str) -> str:
|
|
def replace(match: re.Match[str]) -> str:
|
|
quote = match.group("quote")
|
|
raw = match.group("url")
|
|
local = self._local_reference(local_path, base_url, raw)
|
|
return f"url({quote}{local}{quote})"
|
|
|
|
def replace_import(match: re.Match[str]) -> str:
|
|
quote = match.group("quote")
|
|
local = self._local_reference(local_path, base_url, match.group("url"))
|
|
return f"@import {quote}{local}{quote}"
|
|
|
|
return CSS_IMPORT_RE.sub(replace_import, CSS_URL_RE.sub(replace, text))
|
|
|
|
def _rewrite_javascript(self, text: str, local_path: str, base_url: str) -> str:
|
|
def replace_quoted(match: re.Match[str]) -> str:
|
|
local = self._local_reference(local_path, base_url, match.group("url"))
|
|
prefix = match.group(0)[: match.start("url") - match.start(0)]
|
|
suffix = match.group(0)[match.end("url") - match.start(0) :]
|
|
return f"{prefix}{local}{suffix}"
|
|
|
|
def replace_source_map(match: re.Match[str]) -> str:
|
|
return match.group(0).replace(
|
|
match.group("url"), self._local_reference(local_path, base_url, match.group("url"))
|
|
)
|
|
|
|
text = JS_DYNAMIC_IMPORT_RE.sub(replace_quoted, text)
|
|
text = JS_STATIC_IMPORT_RE.sub(replace_quoted, text)
|
|
text = JS_NEW_URL_RE.sub(replace_quoted, text)
|
|
text = JS_WORKER_RE.sub(replace_quoted, text)
|
|
return JS_SOURCE_MAP_RE.sub(replace_source_map, text)
|
|
|
|
def _local_reference(self, from_path: str, base_url: str, raw_url: str) -> str:
|
|
stripped = raw_url.strip()
|
|
if not stripped or stripped.startswith("#") or self._is_non_fetchable_reference(stripped):
|
|
return raw_url
|
|
absolute = urljoin(base_url, stripped)
|
|
try:
|
|
target_key = canonical_url(absolute, self.rules)
|
|
except UrlPolicyError:
|
|
return raw_url
|
|
target_path = self.url_to_path.get(target_key)
|
|
if not target_path:
|
|
return raw_url
|
|
relative = posixpath.relpath(target_path, start=posixpath.dirname(from_path) or ".")
|
|
fragment = urlsplit(absolute).fragment
|
|
return f"{relative}#{fragment}" if fragment else relative
|
|
|
|
def _rewrite_srcset(self, local_path: str, base_url: str, value: str) -> str:
|
|
candidates: list[str] = []
|
|
for candidate in value.split(","):
|
|
pieces = candidate.strip().split(maxsplit=1)
|
|
if not pieces:
|
|
continue
|
|
rewritten = self._local_reference(local_path, base_url, pieces[0])
|
|
candidates.append(" ".join([rewritten, *pieces[1:]]))
|
|
return ", ".join(candidates)
|
|
|
|
def _build_report(self) -> dict[str, object]:
|
|
elapsed_seconds = round(time.monotonic() - self._started_at, 2)
|
|
quality = "partial" if self._limit_reached or self.errors or self.warnings else "complete"
|
|
return {
|
|
"siteharbor_version": "0.1.0",
|
|
"source_url": self.source_url,
|
|
"entry_point": self._root_record.local_path if self._root_record else None,
|
|
"quality": quality,
|
|
"elapsed_seconds": elapsed_seconds,
|
|
"options": asdict(self.options),
|
|
"stats": self.stats,
|
|
"warnings": self._visible_warnings(),
|
|
"warning_count": self._warning_count,
|
|
"errors": self._visible_errors(),
|
|
"error_count": self._error_count,
|
|
"resources": [record.as_manifest() for record in self.records],
|
|
}
|
|
|
|
def _write_capture_metadata(self, report: dict[str, object]) -> None:
|
|
if not self._site_dir or not self._root_record:
|
|
raise CaptureError("Capture output was not initialized.")
|
|
(self._site_dir / "siteharbor-manifest.json").write_text(
|
|
json.dumps(report, indent=2, ensure_ascii=True), encoding="utf-8"
|
|
)
|
|
readme = "\n".join(
|
|
[
|
|
"SiteHarbor offline capture",
|
|
"",
|
|
f"Entry point: {self._root_record.local_path}",
|
|
f"Source URL: {self.source_url}",
|
|
f"Quality: {report['quality']}",
|
|
"",
|
|
"This is a bounded static capture. Dynamic APIs, authenticated content,",
|
|
"service workers, and runtime-generated behavior may not work offline.",
|
|
"See siteharbor-manifest.json for captured, skipped, and failed resources.",
|
|
"",
|
|
]
|
|
)
|
|
(self._site_dir / "README.txt").write_text(readme, encoding="utf-8")
|
|
|
|
def _emit_progress(self, phase: str, message: str) -> None:
|
|
self.on_progress(phase, message, self.stats.copy(), self._visible_warnings())
|
|
|
|
def _ensure_not_cancelled(self) -> None:
|
|
if self.cancelled():
|
|
raise CaptureCancelled("Capture cancelled")
|
|
|
|
def _ensure_within_duration(self) -> None:
|
|
if self._duration_exceeded():
|
|
raise CaptureError("The configured capture time limit was reached.")
|
|
|
|
def _duration_exceeded(self) -> bool:
|
|
return self._remaining_seconds() <= 0
|
|
|
|
def _remaining_seconds(self) -> float:
|
|
elapsed = time.monotonic() - self._started_at
|
|
return self.options.max_duration_seconds - elapsed
|
|
|
|
def _request_timeout(self) -> httpx.Timeout:
|
|
remaining = max(0.05, self._remaining_seconds())
|
|
return httpx.Timeout(
|
|
connect=min(MAX_CONNECTION_TIMEOUT_SECONDS, remaining),
|
|
read=min(MAX_READ_TIMEOUT_SECONDS, remaining),
|
|
write=min(MAX_CONNECTION_TIMEOUT_SECONDS, remaining),
|
|
pool=min(MAX_CONNECTION_TIMEOUT_SECONDS, remaining),
|
|
)
|
|
|
|
def _pin_target(self, url: str) -> None:
|
|
addresses = resolve_and_validate(url, self.rules)
|
|
parsed = urlsplit(url)
|
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
self._address_book.pin(hostname(url), port, addresses)
|
|
|
|
def _pin_proxy(self) -> None:
|
|
if not self._proxy_url:
|
|
return
|
|
try:
|
|
proxy_url = normalize_url(self._proxy_url, self.rules)
|
|
addresses = resolve_and_validate(proxy_url, self.rules)
|
|
except UrlPolicyError as error:
|
|
raise CaptureError("The configured proxy endpoint is not allowed.") from error
|
|
parsed = urlsplit(proxy_url)
|
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
self._address_book.pin(hostname(proxy_url), port, addresses)
|
|
|
|
@staticmethod
|
|
def _referer_header(referrer: str | None, target_url: str) -> dict[str, str] | None:
|
|
if not referrer:
|
|
return None
|
|
try:
|
|
if not same_site_host(hostname(referrer), hostname(target_url)):
|
|
return None
|
|
except UrlPolicyError:
|
|
return None
|
|
parsed = urlsplit(referrer)
|
|
return {"Referer": f"{parsed.scheme}://{parsed.netloc}/"}
|
|
|
|
def _add_warning(self, message: str) -> None:
|
|
self._warning_count += 1
|
|
if message not in self.warnings:
|
|
if len(self.warnings) < MAX_RECORDED_WARNINGS:
|
|
self.warnings.append(message)
|
|
else:
|
|
self._warnings_omitted += 1
|
|
|
|
def _visible_warnings(self) -> list[str]:
|
|
visible = self.warnings.copy()
|
|
if self._warnings_omitted:
|
|
visible.append(
|
|
f"{self._warnings_omitted} additional warnings were omitted from this report."
|
|
)
|
|
return visible
|
|
|
|
def _visible_errors(self) -> list[dict[str, object]]:
|
|
visible = self.errors.copy()
|
|
if self._errors_omitted:
|
|
visible.append(
|
|
{
|
|
"url": "",
|
|
"kind": "summary",
|
|
"status_code": None,
|
|
"message": f"{self._errors_omitted} additional resource failures were omitted.",
|
|
}
|
|
)
|
|
return visible
|
|
|
|
def _safe_destination(self, local_path: str) -> Path:
|
|
if not self._site_dir:
|
|
raise CaptureError("Capture output was not initialized.")
|
|
destination = self._site_dir.joinpath(*local_path.split("/"))
|
|
resolved_root = self._site_dir.resolve()
|
|
resolved_destination = destination.resolve()
|
|
if not resolved_destination.is_relative_to(resolved_root):
|
|
raise CaptureError("A captured path escaped the archive root.")
|
|
return destination
|
|
|
|
def _resolve_reference(self, base_url: str, raw_url: str) -> str | None:
|
|
value = raw_url.strip()
|
|
if not value or value.startswith("#") or self._is_non_fetchable_reference(value):
|
|
return None
|
|
try:
|
|
return normalize_url(urljoin(base_url, value), self.rules)
|
|
except UrlPolicyError:
|
|
return None
|
|
|
|
@staticmethod
|
|
def _is_non_fetchable_reference(value: str) -> bool:
|
|
return value.lower().startswith(
|
|
("data:", "javascript:", "mailto:", "tel:", "blob:", "about:")
|
|
)
|
|
|
|
@staticmethod
|
|
def _decode_text(body: bytes, encoding: str | None) -> str:
|
|
encodings = [encoding, "utf-8", "utf-8-sig", "latin-1"]
|
|
for candidate in encodings:
|
|
if not candidate:
|
|
continue
|
|
try:
|
|
return body.decode(candidate)
|
|
except (LookupError, UnicodeDecodeError):
|
|
continue
|
|
return body.decode("utf-8", errors="replace")
|
|
|
|
@staticmethod
|
|
def _srcset_urls(value: str) -> list[str]:
|
|
return [
|
|
candidate.strip().split(maxsplit=1)[0]
|
|
for candidate in value.split(",")
|
|
if candidate.strip()
|
|
]
|
|
|
|
@staticmethod
|
|
def _tag_value(tag: Tag, attribute: str) -> str | None:
|
|
value = tag.get(attribute)
|
|
return value if isinstance(value, str) else None
|
|
|
|
@staticmethod
|
|
def _is_html(record: ResourceRecord, text: str | None = None) -> bool:
|
|
if record.content_type in {"text/html", "application/xhtml+xml"}:
|
|
return True
|
|
return bool(text and "<html" in text[:1_024].lower())
|
|
|
|
@staticmethod
|
|
def _is_css(record: ResourceRecord) -> bool:
|
|
return record.content_type == "text/css" or record.local_path.lower().endswith(".css")
|
|
|
|
@staticmethod
|
|
def _is_javascript(record: ResourceRecord) -> bool:
|
|
return "javascript" in record.content_type or record.local_path.lower().endswith(
|
|
(".js", ".mjs", ".cjs")
|
|
)
|