__init__
This commit is contained in:
kstyagi23
@@ -0,0 +1,91 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import zipfile
|
||||
from dataclasses import replace
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from app.capture import CaptureService
|
||||
from app.config import settings
|
||||
from app.db import Database
|
||||
|
||||
|
||||
class MinimalSiteHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
if self.path == "/":
|
||||
body = (
|
||||
b"<html><head><link rel='stylesheet' href='/site.css'></head>"
|
||||
b"<body>Home</body></html>"
|
||||
)
|
||||
content_type = "text/html"
|
||||
status = 200
|
||||
elif self.path == "/site.css":
|
||||
body = b"body { color: #123456; }"
|
||||
content_type = "text/css"
|
||||
status = 200
|
||||
else:
|
||||
body = b"not found"
|
||||
content_type = "text/plain"
|
||||
status = 404
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args) -> None: # type: ignore[override]
|
||||
return
|
||||
|
||||
|
||||
def test_capture_service_creates_private_archive(tmp_path) -> None:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), MinimalSiteHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
port = server.server_address[1]
|
||||
|
||||
test_settings = replace(
|
||||
settings,
|
||||
data_dir=tmp_path,
|
||||
database_path=tmp_path / "siteharbor.sqlite3",
|
||||
work_dir=tmp_path / "work",
|
||||
artifacts_dir=tmp_path / "artifacts",
|
||||
reports_dir=tmp_path / "reports",
|
||||
allow_private_networks=True,
|
||||
allow_nonstandard_ports=True,
|
||||
respect_robots=False,
|
||||
)
|
||||
test_settings.ensure_directories()
|
||||
database = Database(test_settings.database_path)
|
||||
database.initialize()
|
||||
source_url = f"http://127.0.0.1:{port}/"
|
||||
job = database.create_job(
|
||||
source_url,
|
||||
source_url,
|
||||
{
|
||||
"include_external_assets": False,
|
||||
"max_pages": 10,
|
||||
"max_depth": 2,
|
||||
"max_bytes": 2_000_000,
|
||||
"max_duration_seconds": 30,
|
||||
"retention_days": 1,
|
||||
},
|
||||
)
|
||||
claimed = database.claim_next("test-worker", lease_seconds=120)
|
||||
assert claimed is not None
|
||||
|
||||
try:
|
||||
CaptureService(database, test_settings, "test-worker").run(claimed)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
completed = database.get_job(job["id"])
|
||||
assert completed is not None
|
||||
assert completed["state"] == "ready"
|
||||
assert completed["artifact_path"]
|
||||
assert completed["report_path"]
|
||||
|
||||
with zipfile.ZipFile(completed["artifact_path"]) as archive:
|
||||
names = archive.namelist()
|
||||
assert any(name.endswith("siteharbor-manifest.json") for name in names)
|
||||
assert any(name.endswith("site.css") for name in names)
|
||||
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler import CaptureOptions, SiteCrawler
|
||||
|
||||
|
||||
class FixtureHandler(BaseHTTPRequestHandler):
|
||||
fixtures = {
|
||||
"/robots.txt": ("text/plain", b"User-agent: *\nAllow: /\n"),
|
||||
"/": (
|
||||
"text/html",
|
||||
b"""<!doctype html><html><head>
|
||||
<link rel='stylesheet' href='/assets/site.css'>
|
||||
<script src='/assets/app.js'></script>
|
||||
</head><body>
|
||||
<img src='/assets/logo.svg' srcset='/assets/logo.svg 1x, /assets/hero.png 2x'>
|
||||
<a href='/about/'>About</a>
|
||||
</body></html>""",
|
||||
),
|
||||
"/about/": ("text/html", b"<html><body><a href='/'>Home</a></body></html>"),
|
||||
"/assets/site.css": (
|
||||
"text/css",
|
||||
b"@import '/assets/nested.css'; .hero { background: url('/assets/hero.png'); }",
|
||||
),
|
||||
"/assets/nested.css": ("text/css", b".logo { background: url('/assets/logo.svg'); }"),
|
||||
"/assets/app.js": ("application/javascript", b"import('/assets/chunk.js');"),
|
||||
"/assets/chunk.js": ("application/javascript", b"export const loaded = true;"),
|
||||
"/assets/logo.svg": ("image/svg+xml", b"<svg xmlns='http://www.w3.org/2000/svg'></svg>"),
|
||||
"/assets/hero.png": ("image/png", b"not-a-real-png-but-a-static-asset"),
|
||||
}
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
content_type, body = self.fixtures.get(self.path, ("text/plain", b"not found"))
|
||||
status = 200 if self.path in self.fixtures else 404
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args) -> None: # type: ignore[override]
|
||||
return
|
||||
|
||||
|
||||
def test_crawler_captures_and_rewrites_static_fixture(tmp_path) -> None:
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), FixtureHandler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
port = server.server_address[1]
|
||||
|
||||
test_settings = replace(
|
||||
settings,
|
||||
data_dir=tmp_path,
|
||||
work_dir=tmp_path / "work",
|
||||
artifacts_dir=tmp_path / "artifacts",
|
||||
reports_dir=tmp_path / "reports",
|
||||
allow_private_networks=True,
|
||||
allow_nonstandard_ports=True,
|
||||
respect_robots=True,
|
||||
fetch_concurrency=3,
|
||||
)
|
||||
events: list[tuple[str, str]] = []
|
||||
crawler = SiteCrawler(
|
||||
settings=test_settings,
|
||||
options=CaptureOptions(
|
||||
source_url=f"http://127.0.0.1:{port}/",
|
||||
include_external_assets=False,
|
||||
max_pages=10,
|
||||
max_depth=3,
|
||||
max_bytes=2_000_000,
|
||||
max_duration_seconds=30,
|
||||
),
|
||||
cancelled=lambda: False,
|
||||
on_progress=lambda phase, message, stats, warnings: events.append((phase, message)),
|
||||
)
|
||||
|
||||
try:
|
||||
result = crawler.run(tmp_path / "work")
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
host_dir = result.site_dir / "127.0.0.1"
|
||||
root_html = (host_dir / "index.html").read_text(encoding="utf-8")
|
||||
css = (host_dir / "assets" / "site.css").read_text(encoding="utf-8")
|
||||
javascript = (host_dir / "assets" / "app.js").read_text(encoding="utf-8")
|
||||
|
||||
assert result.entry_point == "127.0.0.1/index.html"
|
||||
assert result.stats["pages_fetched"] == 2
|
||||
assert result.stats["assets_fetched"] >= 6
|
||||
assert "about/index.html" in root_html
|
||||
assert "assets/site.css" in root_html
|
||||
assert "hero.png" in css
|
||||
assert "import('chunk.js')" in javascript
|
||||
assert (host_dir / "assets" / "chunk.js").is_file()
|
||||
assert (result.site_dir / "siteharbor-manifest.json").is_file()
|
||||
assert events
|
||||
@@ -0,0 +1,161 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.db import Database
|
||||
|
||||
|
||||
def capture_config() -> dict[str, int | bool]:
|
||||
return {
|
||||
"include_external_assets": True,
|
||||
"max_pages": 20,
|
||||
"max_depth": 3,
|
||||
"max_bytes": 1_000_000,
|
||||
"max_duration_seconds": 60,
|
||||
"retention_days": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_sqlite_lease_claims_one_queued_job(tmp_path) -> None:
|
||||
database = Database(tmp_path / "siteharbor.sqlite3")
|
||||
database.initialize()
|
||||
job = database.create_job(
|
||||
"https://example.com",
|
||||
"https://example.com/",
|
||||
capture_config(),
|
||||
)
|
||||
|
||||
claimed = database.claim_next("worker-a", lease_seconds=30)
|
||||
|
||||
assert claimed is not None
|
||||
assert claimed["id"] == job["id"]
|
||||
assert claimed["state"] == "running"
|
||||
assert database.claim_next("worker-b", lease_seconds=30) is None
|
||||
|
||||
assert database.update_progress(
|
||||
job["id"],
|
||||
"worker-a",
|
||||
30,
|
||||
phase="Packaging",
|
||||
message="Creating archive",
|
||||
stats={
|
||||
"pages_found": 1,
|
||||
"pages_fetched": 1,
|
||||
"assets_found": 0,
|
||||
"assets_fetched": 0,
|
||||
"bytes_downloaded": 10,
|
||||
"files_written": 1,
|
||||
},
|
||||
warnings=[],
|
||||
)
|
||||
assert database.mark_ready(
|
||||
job["id"],
|
||||
"worker-a",
|
||||
artifact_path=str(tmp_path / "capture.zip"),
|
||||
report_path=str(tmp_path / "capture.json"),
|
||||
archive_size=10,
|
||||
entry_point="example.com/index.html",
|
||||
stats={
|
||||
"pages_found": 1,
|
||||
"pages_fetched": 1,
|
||||
"assets_found": 0,
|
||||
"assets_fetched": 0,
|
||||
"bytes_downloaded": 10,
|
||||
"files_written": 1,
|
||||
},
|
||||
warnings=[],
|
||||
retention_days=1,
|
||||
)
|
||||
ready = database.get_job(job["id"])
|
||||
assert ready is not None
|
||||
assert ready["state"] == "ready"
|
||||
assert ready["expires_at"] > ready["completed_at"]
|
||||
|
||||
|
||||
def test_worker_cannot_cancel_without_a_user_cancellation_request(tmp_path) -> None:
|
||||
database = Database(tmp_path / "siteharbor.sqlite3")
|
||||
database.initialize()
|
||||
job = database.create_job(
|
||||
"https://example.com",
|
||||
"https://example.com/",
|
||||
{
|
||||
**capture_config(),
|
||||
"include_external_assets": False,
|
||||
"max_pages": 1,
|
||||
"max_depth": 0,
|
||||
"max_bytes": 1_000,
|
||||
"max_duration_seconds": 30,
|
||||
},
|
||||
)
|
||||
assert database.claim_next("worker-a", lease_seconds=30)
|
||||
|
||||
assert not database.mark_cancelled(job["id"], "worker-a", "Lease was lost")
|
||||
running = database.get_job(job["id"])
|
||||
assert running is not None
|
||||
assert running["state"] == "running"
|
||||
|
||||
database.request_cancel(job["id"])
|
||||
assert database.mark_cancelled(job["id"], "worker-a", "Cancelled by user")
|
||||
cancelled = database.get_job(job["id"])
|
||||
assert cancelled is not None
|
||||
assert cancelled["state"] == "cancelled"
|
||||
|
||||
|
||||
def test_expired_cancel_requested_job_is_terminalized_before_reclaim(tmp_path) -> None:
|
||||
database = Database(tmp_path / "siteharbor.sqlite3")
|
||||
database.initialize()
|
||||
job = database.create_job(
|
||||
"https://example.com",
|
||||
"https://example.com/",
|
||||
{
|
||||
**capture_config(),
|
||||
"include_external_assets": False,
|
||||
"max_pages": 1,
|
||||
"max_depth": 0,
|
||||
"max_bytes": 1_000,
|
||||
"max_duration_seconds": 30,
|
||||
},
|
||||
)
|
||||
assert database.claim_next("worker-a", lease_seconds=30)
|
||||
with database.connection() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE jobs
|
||||
SET cancel_requested = 1, lease_expires_at = '2000-01-01T00:00:00+00:00'
|
||||
WHERE id = ?
|
||||
""",
|
||||
(job["id"],),
|
||||
)
|
||||
|
||||
assert database.claim_next("worker-b", lease_seconds=30) is None
|
||||
cancelled = database.get_job(job["id"])
|
||||
assert cancelled is not None
|
||||
assert cancelled["state"] == "cancelled"
|
||||
|
||||
|
||||
def test_owner_token_scopes_jobs_and_public_stats(tmp_path) -> None:
|
||||
database = Database(tmp_path / "siteharbor.sqlite3")
|
||||
database.initialize()
|
||||
first_owner = "a" * 32
|
||||
second_owner = "b" * 32
|
||||
first = database.create_job(
|
||||
"https://one.example",
|
||||
"https://one.example/",
|
||||
capture_config(),
|
||||
first_owner,
|
||||
)
|
||||
second = database.create_job(
|
||||
"https://two.example",
|
||||
"https://two.example/",
|
||||
capture_config(),
|
||||
second_owner,
|
||||
)
|
||||
|
||||
assert database.get_owned_job(first["id"], first_owner) is not None
|
||||
assert database.get_owned_job(first["id"], second_owner) is None
|
||||
assert [job["id"] for job in database.list_owned_jobs(first_owner)] == [first["id"]]
|
||||
assert [job["id"] for job in database.list_owned_jobs(second_owner)] == [second["id"]]
|
||||
assert database.public_stats() == {
|
||||
"websites_cloned": 0,
|
||||
"bytes_scraped": 0,
|
||||
"active_crawls": 2,
|
||||
"files_archived": 0,
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.main as main
|
||||
from app.config import settings
|
||||
from app.db import Database
|
||||
from app.scheduler import JobRunner
|
||||
|
||||
|
||||
def test_capture_routes_are_scoped_to_the_browser_session(tmp_path, monkeypatch) -> None:
|
||||
test_settings = replace(
|
||||
settings,
|
||||
data_dir=tmp_path,
|
||||
database_path=tmp_path / "siteharbor.sqlite3",
|
||||
work_dir=tmp_path / "work",
|
||||
artifacts_dir=tmp_path / "artifacts",
|
||||
reports_dir=tmp_path / "reports",
|
||||
worker_enabled=False,
|
||||
)
|
||||
test_settings.ensure_directories()
|
||||
test_database = Database(test_settings.database_path)
|
||||
monkeypatch.setattr(main, "settings", test_settings)
|
||||
monkeypatch.setattr(main, "database", test_database)
|
||||
monkeypatch.setattr(main, "runner", JobRunner(test_database, test_settings))
|
||||
|
||||
owner_a = "a" * 32
|
||||
owner_b = "b" * 32
|
||||
with TestClient(main.app) as client:
|
||||
homepage = client.get("/")
|
||||
assert homepage.status_code == 200
|
||||
assert "Copy the web." in homepage.text
|
||||
assert client.get("/api/captures").status_code == 401
|
||||
|
||||
created = client.post(
|
||||
"/api/captures",
|
||||
headers={"X-SiteHarbor-Session": owner_a},
|
||||
json={
|
||||
"url": "https://example.com",
|
||||
"parallel_connections": 2,
|
||||
"download_asset_types": ["styles", "images"],
|
||||
"skip_file_types": ["pdf, zip"],
|
||||
},
|
||||
)
|
||||
assert created.status_code == 202
|
||||
created_job = created.json()
|
||||
job_id = created_job["id"]
|
||||
assert created_job["options"]["parallel_connections"] == 2
|
||||
assert created_job["options"]["download_asset_types"] == ["styles", "images"]
|
||||
assert created_job["options"]["skip_file_types"] == ["pdf", "zip"]
|
||||
|
||||
assert client.get(
|
||||
f"/api/captures/{job_id}", headers={"X-SiteHarbor-Session": owner_b}
|
||||
).status_code == 404
|
||||
assert client.post(
|
||||
f"/api/captures/{job_id}/cancel", headers={"X-SiteHarbor-Session": owner_b}
|
||||
).status_code == 404
|
||||
|
||||
owned_jobs = client.get("/api/captures", headers={"X-SiteHarbor-Session": owner_a})
|
||||
assert owned_jobs.status_code == 200
|
||||
assert [job["id"] for job in owned_jobs.json()["items"]] == [job_id]
|
||||
|
||||
stats = client.get("/api/stats")
|
||||
assert stats.status_code == 200
|
||||
assert stats.json()["active_crawls"] == 1
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.pinned_transport import PinnedAddressBook, PinnedNetworkBackend
|
||||
|
||||
|
||||
def test_pinned_backend_dials_the_validated_address_not_hostname(monkeypatch) -> None:
|
||||
address_book = PinnedAddressBook()
|
||||
address_book.pin("example.com", 443, ["93.184.216.34"])
|
||||
backend = PinnedNetworkBackend(address_book)
|
||||
called: dict[str, object] = {}
|
||||
|
||||
class StubBackend:
|
||||
def connect_tcp(self, host, port, **kwargs):
|
||||
called["host"] = host
|
||||
called["port"] = port
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(backend, "_backend", StubBackend())
|
||||
|
||||
backend.connect_tcp("example.com", 443)
|
||||
|
||||
assert called == {"host": "93.184.216.34", "port": 443}
|
||||
|
||||
|
||||
def test_pinned_backend_rejects_unvalidated_host() -> None:
|
||||
backend = PinnedNetworkBackend(PinnedAddressBook())
|
||||
|
||||
with pytest.raises(Exception, match="No validated address"):
|
||||
backend.connect_tcp("example.com", 443)
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
|
||||
from app.url_policy import NetworkRules, UrlPolicyError, normalize_url, resolve_and_validate
|
||||
|
||||
|
||||
def test_normalize_url_adds_https_and_removes_fragment() -> None:
|
||||
result = normalize_url("example.com/docs#section", NetworkRules())
|
||||
|
||||
assert result == "https://example.com/docs"
|
||||
|
||||
|
||||
def test_normalize_url_preserves_encoded_path_delimiters() -> None:
|
||||
result = normalize_url("https://example.com/a%2Fb", NetworkRules())
|
||||
|
||||
assert result == "https://example.com/a%2Fb"
|
||||
|
||||
|
||||
def test_normalize_url_rejects_credentials_and_unsafe_ports() -> None:
|
||||
with pytest.raises(UrlPolicyError, match="credentials"):
|
||||
normalize_url("https://user:secret@example.com", NetworkRules())
|
||||
|
||||
with pytest.raises(UrlPolicyError, match="ports 80 and 443"):
|
||||
normalize_url("https://example.com:8443", NetworkRules())
|
||||
|
||||
|
||||
def test_resolve_blocks_loopback_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
socket,
|
||||
"getaddrinfo",
|
||||
lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))],
|
||||
)
|
||||
|
||||
with pytest.raises(UrlPolicyError, match="Private, local, and reserved"):
|
||||
resolve_and_validate("https://example.com", NetworkRules())
|
||||
Reference in New Issue
Block a user