205 lines
8.4 KiB
Python
205 lines
8.4 KiB
Python
import asyncio
|
|
import ipaddress
|
|
import shutil
|
|
import socket
|
|
import sys
|
|
from collections.abc import Callable
|
|
from pathlib import Path, PurePosixPath
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
|
|
from app.providers.base import DownloadOptions, DownloadResult, MediaMetadata, ProviderError
|
|
from app.providers.youtube import parse_yt_dlp_progress
|
|
from app.services.url_safety import BROWSER_LIKE_MEDIA_HEADERS, CHROME_WINDOWS_11_USER_AGENT
|
|
|
|
MEDIA_CONTENT_TYPES = (
|
|
"video/",
|
|
"audio/",
|
|
"application/vnd.apple.mpegurl",
|
|
"application/x-mpegurl",
|
|
"application/dash+xml",
|
|
)
|
|
MEDIA_EXTENSIONS = {
|
|
".mp4",
|
|
".mkv",
|
|
".webm",
|
|
".mov",
|
|
".m4v",
|
|
".mp3",
|
|
".m4a",
|
|
".aac",
|
|
".ogg",
|
|
".flac",
|
|
".m3u8",
|
|
".mpd",
|
|
}
|
|
MANIFEST_CONTENT_TYPES = {"application/vnd.apple.mpegurl", "application/x-mpegurl", "application/dash+xml"}
|
|
MANIFEST_EXTENSIONS = {".m3u8", ".mpd"}
|
|
|
|
|
|
def is_stream_manifest(url: str, content_type: str | None = None) -> bool:
|
|
clean_content_type = (content_type or "").split(";", 1)[0].lower()
|
|
ext = PurePosixPath(urlparse(url).path).suffix.lower()
|
|
return clean_content_type in MANIFEST_CONTENT_TYPES or ext in MANIFEST_EXTENSIONS
|
|
|
|
|
|
def ytdlp_command() -> list[str]:
|
|
binary = shutil.which("yt-dlp")
|
|
if binary:
|
|
return [binary]
|
|
return [sys.executable, "-m", "yt_dlp"]
|
|
|
|
|
|
class MediathekProvider:
|
|
name = "mediathek"
|
|
|
|
def can_handle(self, url: str) -> bool:
|
|
parsed = urlparse(url)
|
|
host = parsed.hostname or ""
|
|
excluded_hosts = {"youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "archive.org", "www.archive.org"}
|
|
return parsed.scheme in {"http", "https"} and host not in excluded_hosts
|
|
|
|
async def probe(self, url: str) -> MediaMetadata:
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise ProviderError("Only http/https URLs are allowed")
|
|
self._reject_private_targets(parsed.hostname)
|
|
async with httpx.AsyncClient(follow_redirects=True, timeout=httpx.Timeout(10, read=10), max_redirects=5) as client:
|
|
response = await client.head(url, headers=BROWSER_LIKE_MEDIA_HEADERS)
|
|
if response.status_code in {405, 403}:
|
|
response = await client.get(url, headers={**BROWSER_LIKE_MEDIA_HEADERS, "Range": "bytes=0-0"})
|
|
for hop in [*response.history, response]:
|
|
host = hop.url.host
|
|
if host:
|
|
self._reject_private_targets(host)
|
|
content_type = response.headers.get("content-type", "").split(";", 1)[0].lower()
|
|
ext = PurePosixPath(urlparse(str(response.url)).path).suffix.lower()
|
|
if not (content_type.startswith(MEDIA_CONTENT_TYPES) or ext in MEDIA_EXTENSIONS):
|
|
raise ProviderError(f"URL is reachable, but content type/extension is not accepted: {content_type or ext or 'unknown'}")
|
|
title = PurePosixPath(urlparse(str(response.url)).path).name or str(response.url.host)
|
|
return MediaMetadata(provider=self.name, title=title, external_id=str(response.url))
|
|
|
|
async def download(
|
|
self,
|
|
url: str,
|
|
target_dir: Path,
|
|
options: DownloadOptions,
|
|
progress_callback: Callable[[float], None] | None = None,
|
|
) -> DownloadResult:
|
|
metadata = await self.probe(url)
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
source_url = metadata.external_id or url
|
|
if is_stream_manifest(source_url):
|
|
return await self._download_with_ytdlp(source_url, target_dir, options, progress_callback)
|
|
|
|
name = PurePosixPath(urlparse(source_url).path).name or "download.bin"
|
|
output_path = target_dir / name
|
|
written = 0
|
|
async with httpx.AsyncClient(follow_redirects=True, timeout=httpx.Timeout(20, read=60), max_redirects=5) as client:
|
|
async with client.stream("GET", source_url, headers=BROWSER_LIKE_MEDIA_HEADERS) as response:
|
|
response.raise_for_status()
|
|
for hop in [*response.history, response]:
|
|
host = hop.url.host
|
|
if host:
|
|
self._reject_private_targets(host)
|
|
total = int(response.headers.get("content-length") or 0)
|
|
with output_path.open("wb") as fh:
|
|
async for chunk in response.aiter_bytes():
|
|
written += len(chunk)
|
|
if written > options.max_bytes:
|
|
raise ProviderError("Download exceeds configured maximum size")
|
|
fh.write(chunk)
|
|
if total > 0 and progress_callback is not None:
|
|
progress_callback(min(written / total, 1.0))
|
|
if progress_callback is not None:
|
|
progress_callback(1.0)
|
|
return DownloadResult(output_files=[output_path])
|
|
|
|
async def _download_with_ytdlp(
|
|
self,
|
|
url: str,
|
|
target_dir: Path,
|
|
options: DownloadOptions,
|
|
progress_callback: Callable[[float], None] | None = None,
|
|
) -> DownloadResult:
|
|
"""Download HLS/DASH manifests as real media, not the tiny manifest file.
|
|
|
|
A .m3u8/.mpd is only a playlist/manifest. Saving it directly produces a
|
|
few-KB text file in Jellyfin instead of the actual media stream. Let
|
|
yt-dlp drive ffmpeg so segments are downloaded and remuxed into MP4.
|
|
"""
|
|
command = ytdlp_command()
|
|
output_template = str(target_dir / "%(title,playlist_index,id).120s.%(ext)s")
|
|
height_expr = f"bestvideo[height<={options.max_height}]+bestaudio/best[height<={options.max_height}]/best"
|
|
if options.audio_only:
|
|
height_expr = "bestaudio/best"
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*command,
|
|
"--newline",
|
|
"--no-playlist",
|
|
"--no-warnings",
|
|
"--restrict-filenames",
|
|
"--format",
|
|
height_expr,
|
|
"--remux-video",
|
|
"mp4",
|
|
"--max-filesize",
|
|
str(options.max_bytes),
|
|
"--user-agent",
|
|
CHROME_WINDOWS_11_USER_AGENT,
|
|
"--add-header",
|
|
f"Accept-Language: {BROWSER_LIKE_MEDIA_HEADERS['Accept-Language']}",
|
|
"--output",
|
|
output_template,
|
|
url,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
captured: list[str] = []
|
|
|
|
async def consume(stream: asyncio.StreamReader | None) -> None:
|
|
if stream is None:
|
|
return
|
|
while True:
|
|
line = await stream.readline()
|
|
if not line:
|
|
break
|
|
text = line.decode(errors="replace").strip()
|
|
if not text:
|
|
continue
|
|
captured.append(text)
|
|
if len(captured) > 80:
|
|
del captured[:40]
|
|
progress = parse_yt_dlp_progress(text)
|
|
if progress is not None and progress_callback is not None:
|
|
progress_callback(progress)
|
|
|
|
try:
|
|
await asyncio.wait_for(asyncio.gather(consume(proc.stdout), consume(proc.stderr), proc.wait()), timeout=60 * 60)
|
|
except BaseException:
|
|
if proc.returncode is None:
|
|
proc.kill()
|
|
await proc.wait()
|
|
raise
|
|
if proc.returncode != 0:
|
|
detail = "\n".join(captured).strip() or "yt-dlp manifest download failed"
|
|
raise ProviderError(detail[:1000])
|
|
files = [p for p in target_dir.iterdir() if p.is_file()]
|
|
media_files = [p for p in files if p.suffix.lower() not in {".json", ".jpg", ".jpeg", ".png", ".webp", ".m3u8", ".mpd"}]
|
|
if not media_files:
|
|
raise ProviderError("yt-dlp finished without producing a media file")
|
|
if progress_callback is not None:
|
|
progress_callback(1.0)
|
|
return DownloadResult(output_files=media_files)
|
|
|
|
def _reject_private_targets(self, hostname: str) -> None:
|
|
try:
|
|
infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
|
|
except socket.gaierror as exc:
|
|
raise ProviderError(f"DNS lookup failed for {hostname}") from exc
|
|
for info in infos:
|
|
ip = ipaddress.ip_address(info[4][0])
|
|
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
|
|
raise ProviderError(f"Refusing private or non-public target address for {hostname}")
|