project-kino/nextcloud-import/backend-nextcloud-variant/app/providers/mediathek.py

import asyncio
import ipaddress
import shutil
import socket
from collections.abc import Callable
from pathlib import Path, PurePosixPath
from urllib.parse import urlparse

import httpx

from app.providers.base import DownloadOptions, DownloadResult, MediaMetadata, ProviderError
from app.providers.youtube import parse_yt_dlp_progress

MEDIA_CONTENT_TYPES = (
    "video/", "audio/", "application/vnd.apple.mpegurl", "application/x-mpegurl", "application/dash+xml"
)
MEDIA_EXTENSIONS = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".mp3", ".m4a", ".aac", ".ogg", ".flac", ".m3u8", ".mpd"}


class MediathekProvider:
    name = "mediathek"

    def can_handle(self, url: str) -> bool:
        parsed = urlparse(url)
        host = parsed.hostname or ""
        excluded_hosts = {"youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "archive.org", "www.archive.org"}
        return parsed.scheme in {"http", "https"} and host not in excluded_hosts

    async def probe(self, url: str) -> MediaMetadata:
        parsed = urlparse(url)
        if parsed.scheme not in {"http", "https"} or not parsed.hostname:
            raise ProviderError("Only http/https URLs are allowed")
        self._reject_private_targets(parsed.hostname)
        async with httpx.AsyncClient(follow_redirects=True, timeout=httpx.Timeout(10, read=10), max_redirects=5) as client:
            response = await client.head(url)
            if response.status_code in {405, 403}:
                response = await client.get(url, headers={"Range": "bytes=0-0"})
        for hop in [*response.history, response]:
            host = hop.url.host
            if host:
                self._reject_private_targets(host)
        content_type = self._content_type(response)
        ext = PurePosixPath(urlparse(str(response.url)).path).suffix.lower()
        if not (content_type.startswith(MEDIA_CONTENT_TYPES) or ext in MEDIA_EXTENSIONS):
            raise ProviderError(f"URL is reachable, but content type/extension is not accepted: {content_type or ext or 'unknown'}")
        title = PurePosixPath(urlparse(str(response.url)).path).name or str(response.url.host)
        return MediaMetadata(provider=self.name, title=title, external_id=str(response.url))

    async def download(
        self,
        url: str,
        target_dir: Path,
        options: DownloadOptions,
        progress_callback: Callable[[float], None] | None = None,
    ) -> DownloadResult:
        metadata = await self.probe(url)
        if self._is_manifest(metadata.external_id or url):
            return await self._download_manifest_with_ytdlp(metadata.external_id or url, target_dir, options, progress_callback)

        target_dir.mkdir(parents=True, exist_ok=True)
        name = PurePosixPath(urlparse(metadata.external_id or url).path).name or "download.bin"
        output_path = target_dir / name
        written = 0
        async with httpx.AsyncClient(follow_redirects=True, timeout=httpx.Timeout(20, read=60), max_redirects=5) as client:
            async with client.stream("GET", url) as response:
                response.raise_for_status()
                for hop in [*response.history, response]:
                    host = hop.url.host
                    if host:
                        self._reject_private_targets(host)
                if self._is_manifest(str(response.url), self._content_type(response)):
                    raise ProviderError("Manifest URL must be downloaded through yt-dlp/ffmpeg, not raw HTTP streaming")
                total = int(response.headers.get("content-length") or 0)
                with output_path.open("wb") as fh:
                    async for chunk in response.aiter_bytes():
                        written += len(chunk)
                        if written > options.max_bytes:
                            raise ProviderError("Download exceeds configured maximum size")
                        fh.write(chunk)
                        if total > 0 and progress_callback is not None:
                            progress_callback(min(written / total, 1.0))
                if progress_callback is not None:
                    progress_callback(1.0)
        return DownloadResult(output_files=[output_path])

    async def _download_manifest_with_ytdlp(
        self,
        url: str,
        target_dir: Path,
        options: DownloadOptions,
        progress_callback: Callable[[float], None] | None = None,
    ) -> DownloadResult:
        if shutil.which("yt-dlp") is None:
            raise ProviderError("yt-dlp is required for HLS/DASH manifest downloads")
        if shutil.which("ffmpeg") is None:
            raise ProviderError("ffmpeg is required to remux HLS/DASH manifest downloads")
        target_dir.mkdir(parents=True, exist_ok=True)
        output_template = str(target_dir / "%(title).120s [%(id)s].%(ext)s")
        height_expr = f"bestvideo[height<={options.max_height}]+bestaudio/best[height<={options.max_height}]/best"
        proc = await asyncio.create_subprocess_exec(
            "yt-dlp",
            "--newline",
            "--no-playlist",
            "--no-warnings",
            "--restrict-filenames",
            "--remux-video",
            "mp4",
            "--format",
            height_expr,
            "--output",
            output_template,
            url,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
        )
        captured: list[str] = []

        async def consume(stream: asyncio.StreamReader | None) -> None:
            if stream is None:
                return
            while True:
                line = await stream.readline()
                if not line:
                    break
                text = line.decode(errors="replace").strip()
                if not text:
                    continue
                captured.append(text)
                if len(captured) > 80:
                    del captured[:40]
                progress = parse_yt_dlp_progress(text)
                if progress is not None and progress_callback is not None:
                    progress_callback(progress)

        await asyncio.wait_for(asyncio.gather(consume(proc.stdout), consume(proc.stderr), proc.wait()), timeout=60 * 60)
        if proc.returncode != 0:
            detail = "\n".join(captured).strip() or "yt-dlp manifest download failed"
            raise ProviderError(detail[:1000])
        files = [p for p in target_dir.iterdir() if p.is_file()]
        media_files = [p for p in files if p.suffix.lower() not in {".json", ".jpg", ".jpeg", ".png", ".webp"}]
        if not media_files:
            raise ProviderError("yt-dlp finished without producing a media file")
        for media_file in media_files:
            if media_file.stat().st_size > options.max_bytes:
                raise ProviderError("Downloaded media exceeds configured maximum size")
        if progress_callback is not None:
            progress_callback(1.0)
        return DownloadResult(output_files=media_files)

    def _content_type(self, response: httpx.Response) -> str:
        return response.headers.get("content-type", "").split(";", 1)[0].lower()

    def _is_manifest(self, url: str, content_type: str | None = None) -> bool:
        ext = PurePosixPath(urlparse(url).path).suffix.lower()
        ctype = (content_type or "").lower()
        return ext in {".m3u8", ".mpd"} or ctype in {"application/vnd.apple.mpegurl", "application/x-mpegurl", "application/dash+xml"}

    def _reject_private_targets(self, hostname: str) -> None:
        try:
            infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
        except socket.gaierror as exc:
            raise ProviderError(f"DNS lookup failed for {hostname}") from exc
        for info in infos:
            ip = ipaddress.ip_address(info[4][0])
            if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified:
                raise ProviderError(f"Refusing private or non-public target address for {hostname}")