210 lines
8.1 KiB
Python
210 lines
8.1 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import ipaddress
|
|
import socket
|
|
from dataclasses import dataclass
|
|
from typing import Iterable
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
|
|
|
|
BLOCKED_NETWORKS = [
|
|
ipaddress.ip_network("0.0.0.0/8"),
|
|
ipaddress.ip_network("10.0.0.0/8"),
|
|
ipaddress.ip_network("100.64.0.0/10"),
|
|
ipaddress.ip_network("127.0.0.0/8"),
|
|
ipaddress.ip_network("169.254.0.0/16"),
|
|
ipaddress.ip_network("172.16.0.0/12"),
|
|
ipaddress.ip_network("192.168.0.0/16"),
|
|
ipaddress.ip_network("224.0.0.0/4"),
|
|
ipaddress.ip_network("240.0.0.0/4"),
|
|
ipaddress.ip_network("::1/128"),
|
|
ipaddress.ip_network("::/128"),
|
|
ipaddress.ip_network("fc00::/7"),
|
|
ipaddress.ip_network("fe80::/10"),
|
|
ipaddress.ip_network("ff00::/8"),
|
|
]
|
|
|
|
ALLOWED_CONTENT_PREFIXES = ("video/", "audio/")
|
|
ALLOWED_CONTENT_TYPES = {
|
|
"application/vnd.apple.mpegurl",
|
|
"application/x-mpegurl",
|
|
"application/dash+xml",
|
|
"application/octet-stream", # accepted only with known extension
|
|
}
|
|
ALLOWED_EXTENSIONS = (
|
|
".mp4",
|
|
".m4v",
|
|
".mkv",
|
|
".webm",
|
|
".mov",
|
|
".mp3",
|
|
".m4a",
|
|
".aac",
|
|
".ogg",
|
|
".flac",
|
|
".m3u8",
|
|
".mpd",
|
|
)
|
|
|
|
CHROME_WINDOWS_11_USER_AGENT = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/126.0.0.0 Safari/537.36"
|
|
)
|
|
|
|
# Chrome on Windows 11 intentionally still reports Windows NT 10.0 in the
|
|
# classic User-Agent for compatibility. The client-hint headers below are the
|
|
# browser-like Windows/Chrome signals many upstreams check before serving pages
|
|
# or media manifests. Keep this central so safety checks, analyzer fetches and
|
|
# provider downloads use the same fingerprint.
|
|
BROWSER_LIKE_HEADERS = {
|
|
"User-Agent": CHROME_WINDOWS_11_USER_AGENT,
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,video/*;q=0.8,*/*;q=0.7",
|
|
"Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
"Sec-CH-UA": '"Google Chrome";v="126", "Chromium";v="126", "Not-A.Brand";v="99"',
|
|
"Sec-CH-UA-Mobile": "?0",
|
|
"Sec-CH-UA-Platform": '"Windows"',
|
|
"Upgrade-Insecure-Requests": "1",
|
|
"Sec-Fetch-Dest": "document",
|
|
"Sec-Fetch-Mode": "navigate",
|
|
"Sec-Fetch-Site": "none",
|
|
"Sec-Fetch-User": "?1",
|
|
}
|
|
|
|
BROWSER_LIKE_MEDIA_HEADERS = {
|
|
**BROWSER_LIKE_HEADERS,
|
|
"Accept": "video/webm,video/mp4,video/*;q=0.9,application/vnd.apple.mpegurl;q=0.9,application/dash+xml;q=0.9,*/*;q=0.8",
|
|
"Sec-Fetch-Dest": "video",
|
|
"Sec-Fetch-Mode": "no-cors",
|
|
"Sec-Fetch-Site": "cross-site",
|
|
}
|
|
|
|
|
|
class UnsafeUrlError(ValueError):
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class CheckedUrl:
|
|
final_url: str
|
|
chain: list[str]
|
|
content_type: str | None
|
|
content_length: int | None
|
|
|
|
|
|
def _is_blocked_ip(ip: str) -> bool:
|
|
addr = ipaddress.ip_address(ip)
|
|
return any(addr in net for net in BLOCKED_NETWORKS) or addr.is_private or addr.is_loopback or addr.is_link_local
|
|
|
|
|
|
def validate_scheme(url: str) -> None:
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in {"http", "https"}:
|
|
raise UnsafeUrlError("Nur http/https URLs sind erlaubt")
|
|
if not parsed.hostname:
|
|
raise UnsafeUrlError("URL enthält keinen Hostnamen")
|
|
if parsed.username or parsed.password:
|
|
raise UnsafeUrlError("Zugangsdaten in URLs sind nicht erlaubt")
|
|
|
|
|
|
async def resolve_host(hostname: str, port: int | None = None) -> list[str]:
|
|
def _resolve() -> list[str]:
|
|
infos = socket.getaddrinfo(hostname, port or 443, type=socket.SOCK_STREAM)
|
|
return sorted({str(info[4][0]) for info in infos})
|
|
|
|
return await asyncio.to_thread(_resolve)
|
|
|
|
|
|
async def assert_public_host(url: str) -> None:
|
|
validate_scheme(url)
|
|
parsed = urlparse(url)
|
|
assert parsed.hostname is not None
|
|
try:
|
|
ips = await resolve_host(parsed.hostname, parsed.port)
|
|
except socket.gaierror as exc:
|
|
raise UnsafeUrlError(f"Hostname kann nicht aufgelöst werden: {parsed.hostname}") from exc
|
|
if not ips:
|
|
raise UnsafeUrlError("Hostname hat keine IP-Adressen")
|
|
blocked = [ip for ip in ips if _is_blocked_ip(ip)]
|
|
if blocked:
|
|
raise UnsafeUrlError("URL zeigt auf interne/private IP-Adresse")
|
|
|
|
|
|
def _content_type_allowed(content_type: str | None, url: str) -> bool:
|
|
if not content_type:
|
|
return any(urlparse(url).path.lower().endswith(ext) for ext in ALLOWED_EXTENSIONS)
|
|
clean = content_type.split(";", 1)[0].strip().lower()
|
|
if clean.startswith(ALLOWED_CONTENT_PREFIXES):
|
|
return True
|
|
if clean in ALLOWED_CONTENT_TYPES and any(urlparse(url).path.lower().endswith(ext) for ext in ALLOWED_EXTENSIONS):
|
|
return True
|
|
return False
|
|
|
|
|
|
async def check_url_for_page(url: str, *, timeout: float = 15.0, max_redirects: int = 8) -> CheckedUrl:
|
|
"""Validate URL and redirects for page analysis without requiring media Content-Type."""
|
|
current = str(url)
|
|
chain: list[str] = []
|
|
async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
|
|
for _ in range(max_redirects + 1):
|
|
await assert_public_host(current)
|
|
chain.append(current)
|
|
try:
|
|
response = await client.head(current, headers=BROWSER_LIKE_HEADERS)
|
|
except httpx.TimeoutException:
|
|
response = await client.get(current, headers={**BROWSER_LIKE_MEDIA_HEADERS, "Range": "bytes=0-0"})
|
|
if response.status_code in {405, 403}:
|
|
response = await client.get(current, headers={**BROWSER_LIKE_MEDIA_HEADERS, "Range": "bytes=0-0"})
|
|
if response.is_redirect:
|
|
loc = response.headers.get("location")
|
|
if not loc:
|
|
raise UnsafeUrlError("Redirect ohne Location-Header")
|
|
current = str(response.url.join(loc))
|
|
continue
|
|
response.raise_for_status()
|
|
await assert_public_host(str(response.url))
|
|
content_type = response.headers.get("content-type")
|
|
length_header = response.headers.get("content-length")
|
|
content_length = int(length_header) if length_header and length_header.isdigit() else None
|
|
return CheckedUrl(str(response.url), chain, content_type, content_length)
|
|
raise UnsafeUrlError("Zu viele Weiterleitungen")
|
|
|
|
|
|
async def check_url_for_download(url: str, *, max_bytes: int, timeout: float = 15.0, max_redirects: int = 8) -> CheckedUrl:
|
|
"""Validate URL, every redirect hop, final target, content type, and size.
|
|
|
|
This is the no-domain-allowlist SSRF guard for Mediathek/direct links.
|
|
"""
|
|
current = str(url)
|
|
chain: list[str] = []
|
|
async with httpx.AsyncClient(follow_redirects=False, timeout=timeout) as client:
|
|
for _ in range(max_redirects + 1):
|
|
await assert_public_host(current)
|
|
chain.append(current)
|
|
try:
|
|
response = await client.head(current, headers=BROWSER_LIKE_HEADERS)
|
|
except httpx.TimeoutException:
|
|
response = await client.get(current, headers={**BROWSER_LIKE_MEDIA_HEADERS, "Range": "bytes=0-0"})
|
|
if response.status_code in {405, 403}:
|
|
response = await client.get(current, headers={**BROWSER_LIKE_MEDIA_HEADERS, "Range": "bytes=0-0"})
|
|
if response.is_redirect:
|
|
loc = response.headers.get("location")
|
|
if not loc:
|
|
raise UnsafeUrlError("Redirect ohne Location-Header")
|
|
current = str(response.url.join(loc))
|
|
continue
|
|
response.raise_for_status()
|
|
await assert_public_host(str(response.url))
|
|
content_type = response.headers.get("content-type")
|
|
length_header = response.headers.get("content-length")
|
|
content_length = int(length_header) if length_header and length_header.isdigit() else None
|
|
if content_length is not None and content_length > max_bytes:
|
|
raise UnsafeUrlError("Datei ist größer als das konfigurierte Limit")
|
|
if not _content_type_allowed(content_type, str(response.url)):
|
|
raise UnsafeUrlError("Ziel liefert keinen plausiblen Medien-Content-Type")
|
|
return CheckedUrl(str(response.url), chain, content_type, content_length)
|
|
raise UnsafeUrlError("Zu viele Weiterleitungen")
|