504 lines
20 KiB
Python
504 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import html as html_lib
|
|
import os
|
|
import re
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import unquote, urljoin, urlparse
|
|
|
|
import httpx
|
|
try:
|
|
from yt_dlp import YoutubeDL
|
|
except ImportError: # optional analyzer backend
|
|
YoutubeDL = None # type: ignore[assignment]
|
|
|
|
from app.config import get_settings
|
|
from app.services.url_safety import BROWSER_LIKE_HEADERS, BROWSER_LIKE_MEDIA_HEADERS, CHROME_WINDOWS_11_USER_AGENT, UnsafeUrlError, assert_public_host, check_url_for_download, check_url_for_page
|
|
|
|
_MEDIA_EXTENSIONS = (".mp4", ".m4v", ".webm", ".mkv", ".mov", ".m3u8", ".mpd")
|
|
_MEDIA_CONTENT_TYPES = {
|
|
"video/mp4": "direct_video",
|
|
"video/webm": "direct_video",
|
|
"video/x-matroska": "direct_video",
|
|
"video/mp2t": "hls_segment",
|
|
"application/vnd.apple.mpegurl": "hls_manifest",
|
|
"application/x-mpegurl": "hls_manifest",
|
|
"application/dash+xml": "dash_manifest",
|
|
}
|
|
_QUOTED_URL_RE = re.compile(r"[\"']([^\"']+\.(?:mp4|m4v|webm|mkv|mov|m3u8|mpd)(?:\?[^\"']*)?)[\"']", re.I)
|
|
_ABSOLUTE_MEDIA_URL_RE = re.compile(
|
|
r"(?:https?://|https?:\\/\\/|//|\\/\\/)[^\s\"'<>]+?\.(?:mp4|m4v|webm|mkv|mov|m3u8|mpd)(?=$|[?&#\s\"'<>])(?:[?&][^\s\"'<>]*)?",
|
|
re.I,
|
|
)
|
|
_BROWSER_CAPTURE_LIMIT = 80
|
|
_YTDLP_ALLOWED_HOST_SUFFIXES = (
|
|
"youtube.com",
|
|
"youtu.be",
|
|
"archive.org",
|
|
"ardmediathek.de",
|
|
"zdf.de",
|
|
"arte.tv",
|
|
)
|
|
_AD_HOST_FRAGMENTS = (
|
|
"doubleclick.net",
|
|
"googlesyndication.com",
|
|
"googleadservices.com",
|
|
"adservice.google.",
|
|
"adsystem.com",
|
|
"adnxs.com",
|
|
"adsafeprotected.com",
|
|
"amazon-adsystem.com",
|
|
"taboola.com",
|
|
"outbrain.com",
|
|
"scorecardresearch.com",
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class MediaCandidate:
|
|
url: str
|
|
kind: str
|
|
title: str | None = None
|
|
source: str = "html"
|
|
content_type: str | None = None
|
|
content_length: int | None = None
|
|
file_size: str | None = None
|
|
quality: str | None = None
|
|
allowed: bool = True
|
|
reason: str | None = None
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
class _MediaHTMLParser(HTMLParser):
|
|
def __init__(self, base_url: str) -> None:
|
|
super().__init__()
|
|
self.base_url = base_url
|
|
self.urls: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
attr_map = {key.lower(): value for key, value in attrs if value}
|
|
for attr in ("src", "href", "data-src", "data-url"):
|
|
value = attr_map.get(attr)
|
|
if value and _looks_like_media_url(value):
|
|
self.urls.append(urljoin(self.base_url, value))
|
|
|
|
|
|
def _looks_like_media_url(value: str) -> bool:
|
|
parsed_path = urlparse(value).path.lower()
|
|
return parsed_path.endswith(_MEDIA_EXTENSIONS)
|
|
|
|
|
|
def _looks_like_media_response(url: str, content_type: str | None = None) -> bool:
|
|
clean = (content_type or "").split(";", 1)[0].strip().lower()
|
|
return clean in _MEDIA_CONTENT_TYPES or clean.startswith("video/") or _looks_like_media_url(url)
|
|
|
|
|
|
def human_size(size: int | None) -> str | None:
|
|
if size is None:
|
|
return None
|
|
value = float(size)
|
|
for unit in ("B", "KB", "MB", "GB"):
|
|
if value < 1024 or unit == "GB":
|
|
if unit == "B":
|
|
return f"{int(value)} {unit}"
|
|
return f"{value:.1f} {unit}"
|
|
value /= 1024
|
|
|
|
|
|
def _is_ad_or_tracker_url(url: str) -> bool:
|
|
host = (urlparse(url).hostname or "").lower()
|
|
if any(fragment in host for fragment in _AD_HOST_FRAGMENTS):
|
|
return True
|
|
path = urlparse(url).path.lower()
|
|
return any(marker in path for marker in ("/ads/", "/adserver/", "/prebid", "/vast", "/vpaid"))
|
|
|
|
|
|
def _host_allowed_for_ytdlp(url: str) -> bool:
|
|
host = (urlparse(url).hostname or "").lower().rstrip(".")
|
|
return any(host == suffix or host.endswith(f".{suffix}") for suffix in _YTDLP_ALLOWED_HOST_SUFFIXES)
|
|
|
|
|
|
async def _fetch_public_page_body(url: str, *, timeout: float, max_redirects: int = 8) -> tuple[str, str]:
|
|
"""Fetch a page body while validating every GET redirect hop.
|
|
|
|
``check_url_for_page`` validates HEAD redirects, but some origins redirect
|
|
differently for GET. Keep GET redirects manual so the analyzer cannot be
|
|
turned into an internal-network fetch proxy by a method-dependent redirect.
|
|
"""
|
|
current = url
|
|
async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
|
|
for _ in range(max_redirects + 1):
|
|
checked = await check_url_for_page(current, timeout=timeout, max_redirects=max_redirects)
|
|
response = await client.get(checked.final_url, headers=BROWSER_LIKE_HEADERS)
|
|
if response.is_redirect:
|
|
loc = response.headers.get("location")
|
|
if not loc:
|
|
raise UnsafeUrlError("Redirect ohne Location-Header")
|
|
current = str(response.url.join(loc))
|
|
continue
|
|
response.raise_for_status()
|
|
await assert_public_host(str(response.url))
|
|
return response.text, str(response.url)
|
|
raise UnsafeUrlError("Zu viele Weiterleitungen")
|
|
|
|
|
|
def _ublock_origin_path() -> Path | None:
|
|
configured = os.environ.get("KINOPROJEKT_UBLOCK_PATH")
|
|
candidates = [Path(configured)] if configured else []
|
|
candidates.extend([Path("/opt/kino-projekt/ublock-origin"), Path("/opt/kino-projekt/ublock")])
|
|
for candidate in candidates:
|
|
if not candidate:
|
|
continue
|
|
if candidate.exists() and candidate.is_dir() and (candidate / "manifest.json").exists():
|
|
return candidate
|
|
nested = candidate / "uBlock0.chromium"
|
|
if nested.exists() and nested.is_dir() and (nested / "manifest.json").exists():
|
|
return nested
|
|
return None
|
|
|
|
|
|
def _has_importable_media(candidates: dict[str, tuple[str | None, str | None]]) -> bool:
|
|
return any(
|
|
classify_candidate(url, content_type).kind in {"direct_video", "hls_manifest", "dash_manifest"}
|
|
for url, (_, content_type) in candidates.items()
|
|
)
|
|
|
|
|
|
def _normalize_media_url(raw: str, base_url: str) -> str | None:
|
|
value = html_lib.unescape(raw.strip())
|
|
if not value:
|
|
return None
|
|
value = value.replace("\\/", "/").replace("\\u002F", "/").replace("\\u002f", "/")
|
|
value = value.rstrip(",;)}]")
|
|
if value.startswith("//"):
|
|
value = f"{urlparse(base_url).scheme or 'https'}:{value}"
|
|
return urljoin(base_url, unquote(value))
|
|
|
|
|
|
def extract_media_urls(html: str, base_url: str) -> list[str]:
|
|
parser = _MediaHTMLParser(base_url)
|
|
parser.feed(html)
|
|
urls = list(parser.urls)
|
|
for match in _QUOTED_URL_RE.finditer(html):
|
|
normalized = _normalize_media_url(match.group(1), base_url)
|
|
if normalized:
|
|
urls.append(normalized)
|
|
for match in _ABSOLUTE_MEDIA_URL_RE.finditer(html):
|
|
normalized = _normalize_media_url(match.group(0), base_url)
|
|
if normalized:
|
|
urls.append(normalized)
|
|
# Stable de-duplication while preserving discovery order.
|
|
return list(dict.fromkeys(urls))
|
|
|
|
|
|
def classify_candidate(url: str, content_type: str | None = None) -> MediaCandidate:
|
|
normalized_content_type = (content_type or "").split(";", 1)[0].strip().lower()
|
|
if normalized_content_type in _MEDIA_CONTENT_TYPES:
|
|
kind = _MEDIA_CONTENT_TYPES[normalized_content_type]
|
|
elif normalized_content_type.startswith("video/"):
|
|
kind = "direct_video"
|
|
else:
|
|
path = urlparse(url).path.lower()
|
|
if path.endswith(".m3u8"):
|
|
kind = "hls_manifest"
|
|
elif path.endswith(".mpd"):
|
|
kind = "dash_manifest"
|
|
elif path.endswith((".mp4", ".m4v", ".webm", ".mkv", ".mov")):
|
|
kind = "direct_video"
|
|
else:
|
|
kind = "unknown_media"
|
|
title = unquote(urlparse(url).path.rsplit("/", 1)[-1]) or url
|
|
return MediaCandidate(url=url, kind=kind, title=title, content_type=content_type)
|
|
|
|
|
|
async def _probe_candidate(url: str, source: str, title: str | None = None, quality: str | None = None) -> MediaCandidate:
|
|
settings = get_settings()
|
|
try:
|
|
checked = await check_url_for_download(url, max_bytes=settings.max_download_bytes, timeout=settings.request_timeout)
|
|
candidate = classify_candidate(checked.final_url, checked.content_type)
|
|
candidate.source = source
|
|
candidate.title = title or candidate.title
|
|
candidate.quality = quality
|
|
candidate.content_length = checked.content_length
|
|
candidate.file_size = human_size(checked.content_length)
|
|
candidate.metadata = {"redirect_chain": checked.chain}
|
|
return candidate
|
|
except UnsafeUrlError as exc:
|
|
c = classify_candidate(url)
|
|
c.source = source
|
|
c.title = title or c.title
|
|
c.quality = quality
|
|
c.allowed = False
|
|
c.reason = str(exc)
|
|
return c
|
|
|
|
|
|
async def analyze_html_page(url: str) -> list[MediaCandidate]:
|
|
settings = get_settings()
|
|
# First validate the page URL itself so the analyzer cannot be used as an internal-network proxy.
|
|
checked_page = await check_url_for_page(url, timeout=settings.request_timeout)
|
|
body, final_url = await _fetch_public_page_body(checked_page.final_url, timeout=settings.request_timeout)
|
|
found = extract_media_urls(body, final_url)
|
|
return await _probe_candidates(found, source="html")
|
|
|
|
|
|
async def analyze_with_ytdlp(url: str) -> list[MediaCandidate]:
|
|
checked_page = await check_url_for_page(url, timeout=get_settings().request_timeout)
|
|
|
|
if not _host_allowed_for_ytdlp(checked_page.final_url):
|
|
return []
|
|
|
|
if YoutubeDL is None:
|
|
return []
|
|
|
|
def _extract() -> dict[str, Any] | None:
|
|
opts = {
|
|
"quiet": True,
|
|
"skip_download": True,
|
|
"noplaylist": True,
|
|
"extract_flat": False,
|
|
"socket_timeout": get_settings().request_timeout,
|
|
}
|
|
with YoutubeDL(opts) as ydl:
|
|
return ydl.extract_info(checked_page.final_url, download=False)
|
|
|
|
try:
|
|
info = await asyncio.to_thread(_extract)
|
|
except Exception:
|
|
return []
|
|
if not info:
|
|
return []
|
|
|
|
candidates: list[tuple[str, str | None, str | None]] = []
|
|
title = info.get("title")
|
|
if info.get("url") and _looks_like_media_url(info["url"]):
|
|
candidates.append((info["url"], title, None))
|
|
for fmt in info.get("formats") or []:
|
|
fmt_url = fmt.get("url")
|
|
if not fmt_url:
|
|
continue
|
|
ext = fmt.get("ext") or ""
|
|
protocol = fmt.get("protocol") or ""
|
|
if _looks_like_media_url(fmt_url) or ext in {"mp4", "webm", "m3u8", "mpd"} or "m3u8" in protocol or "dash" in protocol:
|
|
quality = fmt.get("format_note") or fmt.get("format") or fmt.get("height")
|
|
candidates.append((fmt_url, title, str(quality) if quality else None))
|
|
deduped: dict[str, tuple[str | None, str | None]] = {}
|
|
for candidate_url, candidate_title, quality in candidates:
|
|
deduped.setdefault(candidate_url, (candidate_title, quality))
|
|
return await _probe_candidates_with_meta(deduped, source="yt-dlp")
|
|
|
|
|
|
async def _trigger_video_playback(page) -> None:
|
|
"""Try to start embedded players so lazy video requests become visible."""
|
|
await page.evaluate(
|
|
"""
|
|
async () => {
|
|
for (const text of ['Akzeptieren', 'Accept', 'I agree', 'OK']) {
|
|
const buttons = [...document.querySelectorAll('button, [role="button"], input[type="button"], input[type="submit"]')];
|
|
const button = buttons.find(el => (el.innerText || el.value || '').toLowerCase().includes(text.toLowerCase()));
|
|
if (button) { try { button.click(); } catch (_) {} }
|
|
}
|
|
for (const video of document.querySelectorAll('video')) {
|
|
try {
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.autoplay = true;
|
|
video.play().catch(()=>{});
|
|
} catch (_) {}
|
|
}
|
|
const playSelectors = [
|
|
'[aria-label*="play" i]', '[title*="play" i]', '.play', '.play-button', '.vjs-big-play-button',
|
|
'button[class*="play" i]', 'button[aria-label*="abspielen" i]'
|
|
];
|
|
for (const selector of playSelectors) {
|
|
for (const el of document.querySelectorAll(selector)) {
|
|
try { el.click(); } catch (_) {}
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
)
|
|
|
|
|
|
async def _browser_context(playwright, *, ignore_https_errors: bool = False):
|
|
base_args = [
|
|
"--disable-dev-shm-usage",
|
|
"--disable-gpu",
|
|
"--no-sandbox",
|
|
"--disable-setuid-sandbox",
|
|
"--disable-background-networking",
|
|
"--window-size=1365,768",
|
|
]
|
|
context_options = {
|
|
"ignore_https_errors": ignore_https_errors,
|
|
"java_script_enabled": True,
|
|
"user_agent": CHROME_WINDOWS_11_USER_AGENT,
|
|
"locale": "de-DE",
|
|
"timezone_id": "Europe/Berlin",
|
|
"viewport": {"width": 1365, "height": 768},
|
|
"device_scale_factor": 1,
|
|
"is_mobile": False,
|
|
"has_touch": False,
|
|
"extra_http_headers": {
|
|
key: value for key, value in BROWSER_LIKE_HEADERS.items() if key.lower() != "user-agent"
|
|
},
|
|
}
|
|
ublock_path = _ublock_origin_path()
|
|
if ublock_path:
|
|
user_data_dir = tempfile.TemporaryDirectory(prefix="kino-browser-")
|
|
context = await playwright.chromium.launch_persistent_context(
|
|
user_data_dir.name,
|
|
headless=True,
|
|
args=[
|
|
*base_args,
|
|
f"--disable-extensions-except={ublock_path}",
|
|
f"--load-extension={ublock_path}",
|
|
],
|
|
**context_options,
|
|
)
|
|
return context, None, user_data_dir, str(ublock_path)
|
|
|
|
browser = await playwright.chromium.launch(headless=True, args=base_args)
|
|
context = await browser.new_context(**context_options)
|
|
return context, browser, None, None
|
|
|
|
|
|
async def analyze_with_browser(url: str) -> list[MediaCandidate]:
|
|
"""Headless-Browser-Analyzer (Option B): observe JS-loaded media requests.
|
|
|
|
Every browser request is still constrained to public http(s) targets. This keeps the
|
|
server-side browser from becoming a LAN/internal-network fetch proxy.
|
|
"""
|
|
settings = get_settings()
|
|
checked_page = await check_url_for_page(url, timeout=settings.request_timeout)
|
|
try:
|
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
|
from playwright.async_api import async_playwright
|
|
except ImportError:
|
|
return []
|
|
|
|
seen: dict[str, tuple[str | None, str | None]] = {}
|
|
playback_triggered = False
|
|
|
|
async with async_playwright() as p:
|
|
context, browser, user_data_dir, ublock_path = await _browser_context(p, ignore_https_errors=False)
|
|
|
|
async def guard_route(route):
|
|
request_url = route.request.url
|
|
parsed = urlparse(request_url)
|
|
if parsed.scheme not in {"http", "https"}:
|
|
await route.abort()
|
|
return
|
|
if _is_ad_or_tracker_url(request_url):
|
|
await route.abort()
|
|
return
|
|
try:
|
|
await assert_public_host(request_url)
|
|
except UnsafeUrlError:
|
|
await route.abort()
|
|
return
|
|
await route.continue_()
|
|
|
|
async def on_response(response):
|
|
response_url = response.url
|
|
headers = await response.all_headers()
|
|
content_type = headers.get("content-type")
|
|
if _looks_like_media_response(response_url, content_type):
|
|
title = unquote(urlparse(response_url).path.rsplit("/", 1)[-1]) or response_url
|
|
seen.setdefault(response_url, (title, content_type))
|
|
|
|
await context.route("**/*", guard_route)
|
|
pending_response_tasks: set[asyncio.Task] = set()
|
|
|
|
def schedule_response_probe(response):
|
|
task = asyncio.create_task(on_response(response))
|
|
pending_response_tasks.add(task)
|
|
task.add_done_callback(pending_response_tasks.discard)
|
|
|
|
context.on("response", schedule_response_probe)
|
|
page = await context.new_page()
|
|
try:
|
|
await page.goto(checked_page.final_url, wait_until="domcontentloaded", timeout=int(settings.request_timeout * 1000))
|
|
try:
|
|
await page.wait_for_load_state("networkidle", timeout=5000)
|
|
except PlaywrightTimeoutError:
|
|
pass
|
|
if pending_response_tasks:
|
|
await asyncio.gather(*pending_response_tasks, return_exceptions=True)
|
|
if not _has_importable_media(seen):
|
|
playback_triggered = True
|
|
await _trigger_video_playback(page)
|
|
try:
|
|
await page.wait_for_load_state("networkidle", timeout=5000)
|
|
except PlaywrightTimeoutError:
|
|
pass
|
|
if pending_response_tasks:
|
|
await asyncio.gather(*pending_response_tasks, return_exceptions=True)
|
|
html = await page.content()
|
|
for dom_url in extract_media_urls(html, page.url):
|
|
seen.setdefault(dom_url, (None, None))
|
|
finally:
|
|
await context.close()
|
|
if browser is not None:
|
|
await browser.close()
|
|
if user_data_dir is not None:
|
|
user_data_dir.cleanup()
|
|
|
|
candidates = await _probe_candidates_with_meta(dict(list(seen.items())[:_BROWSER_CAPTURE_LIMIT]), source="browser")
|
|
for candidate in candidates:
|
|
candidate.metadata["playback_triggered"] = playback_triggered
|
|
candidate.metadata["ublock_origin"] = bool(ublock_path)
|
|
return candidates
|
|
|
|
|
|
async def analyze_browser_capture(captured: list[dict[str, Any]], page_url: str | None = None) -> list[MediaCandidate]:
|
|
"""Validate and normalize media URLs captured by a user's real browser (Option C)."""
|
|
base_url = page_url or ""
|
|
deduped: dict[str, tuple[str | None, str | None]] = {}
|
|
for item in captured[:_BROWSER_CAPTURE_LIMIT]:
|
|
raw_url = str(item.get("url") or "").strip()
|
|
if not raw_url:
|
|
continue
|
|
candidate_url = urljoin(base_url, raw_url) if base_url else raw_url
|
|
if not _looks_like_media_response(candidate_url, item.get("content_type") or item.get("type")):
|
|
continue
|
|
title = item.get("title") or item.get("name")
|
|
quality = item.get("quality")
|
|
deduped.setdefault(candidate_url, (str(title) if title else None, str(quality) if quality else None))
|
|
return await _probe_candidates_with_meta(deduped, source="browser-capture")
|
|
|
|
|
|
async def _probe_candidates(urls: list[str], source: str) -> list[MediaCandidate]:
|
|
return await _probe_candidates_with_meta({url: (None, None) for url in urls}, source=source)
|
|
|
|
|
|
async def _probe_candidates_with_meta(urls: dict[str, tuple[str | None, str | None]], source: str) -> list[MediaCandidate]:
|
|
candidates = [await _probe_candidate(url, source, title, quality) for url, (title, quality) in list(urls.items())[:50]]
|
|
if any(candidate.kind in {"hls_manifest", "dash_manifest"} for candidate in candidates):
|
|
candidates = [candidate for candidate in candidates if candidate.kind != "hls_segment"]
|
|
# Show usable candidates first, then blocked candidates as diagnostics.
|
|
candidates.sort(key=lambda c: (not c.allowed, c.kind, c.quality or ""))
|
|
return candidates
|
|
|
|
|
|
async def analyze_page(url: str) -> list[MediaCandidate]:
|
|
results = await asyncio.gather(
|
|
analyze_html_page(url),
|
|
analyze_with_ytdlp(url),
|
|
analyze_with_browser(url),
|
|
return_exceptions=True,
|
|
)
|
|
html_candidates, ytdlp_candidates, browser_candidates = [item if isinstance(item, list) else [] for item in results]
|
|
merged: dict[str, MediaCandidate] = {}
|
|
# Prefer browser/ytdlp metadata over static HTML when the same URL appears twice.
|
|
for candidate in [*browser_candidates, *ytdlp_candidates, *html_candidates]:
|
|
existing = merged.get(candidate.url)
|
|
if not existing or (not existing.allowed and candidate.allowed):
|
|
merged[candidate.url] = candidate
|
|
return list(merged.values())
|