Merge branch 'main' into codex/fetch-proxies-from-proxyscrape-api-4vaqb3

Use ProxyScrape API for SOCKS lists and normalize single-line responses

docker-compose.yml

@@ -26,6 +26,6 @@ services:
     environment:
       TZ: Europe/Berlin
     volumes:
-      - ./data/jd-output:/output:ro
+      - ./data/jd-output:/output:rw
       - ./data/md5:/md5:rw
       - /root/.ssh/id_ed25519:/ssh/id_ed25519:ro

jd-webgui/app.py

@@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from myjdapi import Myjdapi
 import paramiko
 from fastapi import FastAPI, Form, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 
 # ============================================================
@@ -56,9 +56,13 @@ POLL_SECONDS = float(os.environ.get("POLL_SECONDS", "5"))
 
 # JDownloader writes here inside container
 JD_OUTPUT_PATH = "/output"
+PROXY_EXPORT_PATH = os.environ.get("PROXY_EXPORT_PATH", "/output/jd-proxies.jdproxies")
+LOG_BUFFER_LIMIT = int(os.environ.get("LOG_BUFFER_LIMIT", "500"))
 
 URL_RE = re.compile(r"^https?://", re.I)
 
+NO_PROXY_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
 VIDEO_EXTS = {
     ".mkv", ".mp4", ".m4v", ".avi", ".mov", ".wmv", ".flv", ".webm",
     ".ts", ".m2ts", ".mts", ".mpg", ".mpeg", ".vob", ".ogv",
@@ -122,6 +126,21 @@ class Job:
 
 jobs: Dict[str, Job] = {}
 lock = threading.Lock()
+log_lock = threading.Lock()
+connection_logs: List[str] = []
+
+def log_connection(message: str) -> None:
+    timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    line = f"[{timestamp}] {message}"
+    with log_lock:
+        connection_logs.append(line)
+        if len(connection_logs) > LOG_BUFFER_LIMIT:
+            excess = len(connection_logs) - LOG_BUFFER_LIMIT
+            del connection_logs[:excess]
+
+def get_connection_logs() -> str:
+    with log_lock:
+        return "\n".join(connection_logs)
 
 # ============================================================
 # Core helpers
@@ -148,6 +167,7 @@ def ensure_env():
 
 def get_device():
     jd = Myjdapi()
+    log_connection(f"MyJDownloader connect as {MYJD_EMAIL or 'unknown'}")
     jd.connect(MYJD_EMAIL, MYJD_PASSWORD)
 
     wanted = (MYJD_DEVICE or "").strip()
@@ -245,6 +265,7 @@ def ffprobe_ok(path: str) -> bool:
 def ssh_connect() -> paramiko.SSHClient:
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    log_connection(f"SSH connect {JELLYFIN_USER}@{JELLYFIN_HOST}:{JELLYFIN_PORT}")
     ssh.connect(
         hostname=JELLYFIN_HOST,
         port=JELLYFIN_PORT,
@@ -267,6 +288,7 @@ def sftp_mkdirs(sftp: paramiko.SFTPClient, remote_dir: str):
 def sftp_upload(ssh: paramiko.SSHClient, local_path: str, remote_path: str):
     sftp = ssh.open_sftp()
     try:
+        log_connection(f"SFTP upload {local_path} -> {remote_path}")
         sftp_mkdirs(sftp, os.path.dirname(remote_path))
         sftp.put(local_path, remote_path)
     finally:
@@ -275,6 +297,7 @@ def sftp_upload(ssh: paramiko.SSHClient, local_path: str, remote_path: str):
 def remote_md5sum(ssh: paramiko.SSHClient, remote_path: str) -> str:
     quoted = shlex.quote(remote_path)
     cmd = f"md5sum {quoted}"
+    log_connection(f"SSH exec {cmd}")
     stdin, stdout, stderr = ssh.exec_command(cmd, timeout=120)
     out = stdout.read().decode("utf-8", "replace").strip()
     err = stderr.read().decode("utf-8", "replace").strip()
@@ -289,7 +312,8 @@ def remote_md5sum(ssh: paramiko.SSHClient, remote_path: str) -> str:
 # ============================================================
 def _http_get_json(url: str, headers: Optional[Dict[str, str]] = None) -> Any:
     req = urllib.request.Request(url, headers=headers or {})
-    with urllib.request.urlopen(req, timeout=20) as r:
+    log_connection(f"HTTP GET {url} (no-proxy)")
+    with NO_PROXY_OPENER.open(req, timeout=20) as r:
         return json.loads(r.read().decode("utf-8", "replace"))
 
 def tmdb_search_movie(query: str) -> Optional[Dict[str, Any]]:
@@ -321,6 +345,147 @@ def sanitize_name(name: str) -> str:
     out = "".join("_" if c in bad else c for c in name).strip()
     return re.sub(r"\s+", " ", out)
 
+def format_proxy_lines(raw: str, scheme: str) -> str:
+    """
+    Takes raw lines (ip:port or scheme://ip:port) and outputs normalized lines:
+    scheme://ip:port (one per line). Ignores empty lines and comments.
+    """
+    scheme = scheme.strip().lower()
+    if scheme not in {"socks5", "socks4", "http"}:
+        raise ValueError("Unsupported proxy scheme")
+
+    out = []
+    for line in (raw or "").splitlines():
+        s = line.strip()
+        if not s or s.startswith("#"):
+            continue
+
+        if "://" in s:
+            s = s.split("://", 1)[1].strip()
+
+        if ":" not in s:
+            continue
+
+        host, port = s.rsplit(":", 1)
+        host = host.strip()
+        port = port.strip()
+
+        if not host or not port.isdigit():
+            continue
+
+        out.append(f"{scheme}://{host}:{port}")
+
+    seen = set()
+    dedup = []
+    for x in out:
+        if x not in seen:
+            seen.add(x)
+            dedup.append(x)
+
+    return "\n".join(dedup)
+
+def fetch_proxy_list(url: str) -> str:
+    req = urllib.request.Request(url)
+    log_connection(f"HTTP GET {url} (no-proxy)")
+    with NO_PROXY_OPENER.open(req, timeout=20) as resp:
+        text = resp.read().decode("utf-8", "replace")
+    if "\n" not in text and re.search(r"\s", text):
+        return re.sub(r"\s+", "\n", text.strip())
+    return text
+
+def build_jdproxies_payload(text: str) -> Dict[str, Any]:
+    if not text.strip():
+        raise ValueError("Keine Proxy-Einträge zum Speichern.")
+    blacklist_filter = {
+        "entries": [
+            "# Dies ist ein Kommentar",
+            "// Dies ist auch ein Kommentar",
+            "# Für jdownloader.org auskommentieren",
+            "# jdownloader.org",
+            "# unten für alle Accounts mit der ID 'test *' @ jdownloader.org auskommentieren",
+            "#test@jdownloader.org",
+            "# Kommentar unten für ein Konto mit der ID 'test' @ jdownloader.org",
+            "#test$@jdownloader.org",
+            "# Sie können Muster für Konto-ID und Host verwenden, z. B. accountPattern @ hostPattern",
+            "",
+            "my.jdownloader.org",
+            "",
+            "api.jdownloader.org",
+            "",
+            "*.jdownloader.org",
+            "",
+            "*.your-server.de",
+            "88.99.115.46",
+        ],
+        "type": "BLACKLIST",
+    }
+    entries: List[Dict[str, Any]] = []
+    type_map = {
+        "socks5": "SOCKS5",
+        "socks4": "SOCKS4",
+        "http": "HTTP",
+    }
+    entries.append({
+        "filter": None,
+        "proxy": {
+            "address": None,
+            "password": None,
+            "port": 80,
+            "type": "NONE",
+            "username": None,
+            "connectMethodPrefered": False,
+            "preferNativeImplementation": False,
+            "resolveHostName": False,
+        },
+        "enabled": True,
+        "pac": False,
+        "rangeRequestsSupported": True,
+        "reconnectSupported": True,
+    })
+    for line in text.splitlines():
+        s = line.strip()
+        if not s:
+            continue
+        parsed = urllib.parse.urlparse(s)
+        if not parsed.scheme or not parsed.hostname or parsed.port is None:
+            continue
+        proxy_type = type_map.get(parsed.scheme.lower())
+        if not proxy_type:
+            continue
+        entries.append({
+            "filter": blacklist_filter,
+            "proxy": {
+                "address": parsed.hostname,
+                "password": None,
+                "port": int(parsed.port),
+                "type": proxy_type,
+                "username": None,
+                "connectMethodPrefered": False,
+                "preferNativeImplementation": False,
+                "resolveHostName": False,
+            },
+            "enabled": True,
+            "pac": False,
+            "rangeRequestsSupported": True,
+            "reconnectSupported": False,
+        })
+    if not entries:
+        raise ValueError("Keine gültigen Proxy-Einträge gefunden.")
+    return {"customProxyList": entries}
+
+def save_proxy_export(text: str) -> str:
+    payload = build_jdproxies_payload(text)
+    export_path = PROXY_EXPORT_PATH
+    export_dir = os.path.dirname(export_path)
+    if export_dir:
+        os.makedirs(export_dir, exist_ok=True)
+    if os.path.exists(export_path):
+        os.remove(export_path)
+    with open(export_path, "w", encoding="utf-8") as handle:
+        handle.write(json.dumps(payload, indent=2))
+        handle.write("\n")
+    return export_path
+
 def pick_library_target(library_choice: str, filename: str, package_name: str) -> str:
     if library_choice not in {"movies", "series", "auto"}:
         library_choice = "auto"
@@ -392,7 +557,8 @@ def jellyfin_refresh_library():
         try:
             url = JELLYFIN_API_BASE + path
             req = urllib.request.Request(url, headers=headers, method="POST")
-            with urllib.request.urlopen(req, timeout=20) as r:
+            log_connection(f"HTTP POST {url} (no-proxy)")
+            with NO_PROXY_OPENER.open(req, timeout=20) as r:
                 _ = r.read()
             return
         except Exception:
@@ -686,7 +852,19 @@ def worker(jobid: str):
 def favicon():
     return HTMLResponse(status_code=204)
 
-def render_page(error: str = "") -> str:
+@app.get("/jobs", response_class=HTMLResponse)
+def jobs_get():
+    return HTMLResponse(render_job_rows())
+
+@app.get("/logs", response_class=HTMLResponse)
+def logs_get():
+    return HTMLResponse(render_logs_page())
+
+@app.get("/logs/data", response_class=PlainTextResponse)
+def logs_data():
+    return PlainTextResponse(get_connection_logs())
+
+def render_job_rows() -> str:
     rows = ""
     with lock:
         job_list = list(jobs.values())[::-1]
@@ -716,6 +894,13 @@ def render_page(error: str = "") -> str:
             f"</tr>"
         )
 
+    if not rows:
+        rows = "<tr><td colspan='5'><em>No jobs yet.</em></td></tr>"
+    return rows
+
+def render_page(error: str = "") -> str:
+    rows = render_job_rows()
+
     err_html = f"<p class='error'>{error}</p>" if error else ""
     auth_note = "aktiv" if _auth_enabled() else "aus"
     return f"""
@@ -725,14 +910,23 @@ def render_page(error: str = "") -> str:
       <meta charset="utf-8">
       <title>JD → Jellyfin</title>
       <script>
-        setInterval(() => {{
+        async function refreshJobs() {{
           if (document.hidden) return;
-          window.location.reload();
+          try {{
-        }}, 5000);
+            const resp = await fetch('/jobs');
+            if (!resp.ok) return;
+            const html = await resp.text();
+            const tbody = document.getElementById('jobs-body');
+            if (tbody) tbody.innerHTML = html;
+          }} catch (e) {{
+          }}
+        }}
+        setInterval(refreshJobs, 5000);
       </script>
     </head>
     <body>
       <h1>JD → Jellyfin</h1>
+      {render_nav("downloads")}
       {err_html}
 
       <form method="post" action="/submit">
@@ -765,14 +959,121 @@ def render_page(error: str = "") -> str:
         <thead>
           <tr><th>JobID</th><th>URL</th><th>Paket</th><th>Ziel</th><th>Status</th></tr>
         </thead>
-        <tbody>
+        <tbody id="jobs-body">
-          {rows if rows else "<tr><td colspan='5'><em>No jobs yet.</em></td></tr>"}
+          {rows}
         </tbody>
       </table>
     </body>
     </html>
     """
 
+def render_nav(active: str) -> str:
+    def link(label: str, href: str, key: str) -> str:
+        style = "font-weight:700;" if active == key else ""
+        return f"<a href='{href}' style='margin-right:14px; {style}'>{label}</a>"
+    return (
+        "<div style='margin: 8px 0 14px 0;'>"
+        + link("Downloads", "/", "downloads")
+        + link("Proxies", "/proxies", "proxies")
+        + link("Logs", "/logs", "logs")
+        + "</div>"
+    )
+
+def render_logs_page() -> str:
+    return f"""
+    <html>
+    <head>
+      <link rel="stylesheet" href="/static/style.css">
+      <meta charset="utf-8">
+      <title>JD → Jellyfin (Logs)</title>
+      <script>
+        async function refreshLogs() {{
+          if (document.hidden) return;
+          try {{
+            const resp = await fetch('/logs/data');
+            if (!resp.ok) return;
+            const text = await resp.text();
+            const area = document.getElementById('log-body');
+            if (area) {{
+              area.value = text;
+              area.scrollTop = area.scrollHeight;
+            }}
+          }} catch (e) {{
+          }}
+        }}
+        setInterval(refreshLogs, 2000);
+        window.addEventListener('load', refreshLogs);
+      </script>
+    </head>
+    <body>
+      <h1>JD → Jellyfin</h1>
+      {render_nav("logs")}
+      <p class="hint">Verbindungs-Debugger (Echtzeit). Letzte {LOG_BUFFER_LIMIT} Einträge.</p>
+      <textarea id="log-body" class="log-area" rows="20" readonly></textarea>
+    </body>
+    </html>
+    """
+
+def render_proxies_page(
+    error: str = "",
+    message: str = "",
+    socks5_in: str = "",
+    socks4_in: str = "",
+    out_text: str = "",
+    export_path: str = "",
+) -> str:
+    err_html = f"<p class='error'>{error}</p>" if error else ""
+    msg_html = f"<p class='success'>{message}</p>" if message else ""
+    return f"""
+    <html>
+    <head>
+      <link rel="stylesheet" href="/static/style.css">
+      <meta charset="utf-8">
+      <title>JD → Jellyfin (Proxies)</title>
+    </head>
+    <body>
+      <h1>JD → Jellyfin</h1>
+      {render_nav("proxies")}
+      {err_html}
+      {msg_html}
+
+      <form method="post" action="/proxies">
+        <div class="row">
+          <label>SOCKS5 (ein Proxy pro Zeile, z. B. IP:PORT)</label><br/>
+          <textarea name="socks5_in" rows="6" style="width:100%; max-width:860px; padding:10px; border:1px solid #ccc; border-radius:8px;">{socks5_in}</textarea>
+        </div>
+
+        <div class="row">
+          <label>SOCKS4 (ein Proxy pro Zeile, z. B. IP:PORT)</label><br/>
+          <textarea name="socks4_in" rows="6" style="width:100%; max-width:860px; padding:10px; border:1px solid #ccc; border-radius:8px;">{socks4_in}</textarea>
+        </div>
+
+        <button type="submit">In JDownloader-Format umwandeln</button>
+      </form>
+
+      <h2 style="margin-top:18px;">JDownloader Import-Liste</h2>
+      <p class="hint">Format: <code>socks5://IP:PORT</code>, <code>socks4://IP:PORT</code>. Keine Prüfung/Validierung.</p>
+
+      <div class="row">
+        <textarea id="out" rows="12" readonly style="width:100%; max-width:860px; padding:10px; border:1px solid #ccc; border-radius:8px;">{out_text}</textarea>
+      </div>
+
+      <button type="button" onclick="navigator.clipboard.writeText(document.getElementById('out').value)">Kopieren</button>
+
+      <h2 style="margin-top:18px;">Datei für Connection Manager</h2>
+      <p class="hint">Speichert die Liste als <code>.jdproxies</code> im Container, z. B. zum Import in JDownloader → Verbindungsmanager → Importieren.</p>
+
+      <form method="post" action="/proxies/save">
+        <textarea name="socks5_in" style="display:none;">{socks5_in}</textarea>
+        <textarea name="socks4_in" style="display:none;">{socks4_in}</textarea>
+        <button type="submit">Liste als JDProxies speichern</button>
+      </form>
+
+      <p class="hint">Aktueller Pfad: <code>{export_path or PROXY_EXPORT_PATH}</code></p>
+    </body>
+    </html>
+    """
+
 @app.get("/", response_class=HTMLResponse)
 def index():
     try:
@@ -830,3 +1131,77 @@ def cancel(jobid: str):
         job.cancel_requested = True
         job.message = "Abbruch angefordert…"
     return RedirectResponse(url="/", status_code=303)
+
+@app.get("/proxies", response_class=HTMLResponse)
+def proxies_get():
+    try:
+        socks5_in = fetch_proxy_list(
+            "https://api.proxyscrape.com/v4/free-proxy-list/get?request=displayproxies&protocol=socks5&timeout=10000&country=all&ssl=yes&anonymity=elite&skip=0&limit=2000"
+        )
+        socks4_in = fetch_proxy_list(
+            "https://api.proxyscrape.com/v4/free-proxy-list/get?request=displayproxies&protocol=socks4&timeout=10000&country=all&ssl=yes&anonymity=elite&skip=0&limit=2000"
+        )
+        http_in = fetch_proxy_list("https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt")
+
+        s5 = format_proxy_lines(socks5_in, "socks5")
+        s4 = format_proxy_lines(socks4_in, "socks4")
+        combined = "\n".join([x for x in [s5, s4] if x.strip()])
+        return HTMLResponse(render_proxies_page(
+            socks5_in=socks5_in,
+            socks4_in=socks4_in,
+            out_text=combined,
+            export_path=PROXY_EXPORT_PATH,
+        ))
+    except Exception as e:
+        return HTMLResponse(render_proxies_page(error=str(e)), status_code=502)
+
+@app.post("/proxies", response_class=HTMLResponse)
+def proxies_post(
+    socks5_in: str = Form(""),
+    socks4_in: str = Form(""),
+):
+    try:
+        s5 = format_proxy_lines(socks5_in, "socks5")
+        s4 = format_proxy_lines(socks4_in, "socks4")
+
+        combined = "\n".join([x for x in [s5, s4] if x.strip()])
+        return HTMLResponse(render_proxies_page(
+            socks5_in=socks5_in,
+            socks4_in=socks4_in,
+            out_text=combined,
+            export_path=PROXY_EXPORT_PATH,
+        ))
+    except Exception as e:
+        return HTMLResponse(render_proxies_page(
+            error=str(e),
+            socks5_in=socks5_in,
+            socks4_in=socks4_in,
+            out_text="",
+            export_path=PROXY_EXPORT_PATH,
+        ), status_code=400)
+
+@app.post("/proxies/save", response_class=HTMLResponse)
+def proxies_save(
+    socks5_in: str = Form(""),
+    socks4_in: str = Form(""),
+):
+    try:
+        s5 = format_proxy_lines(socks5_in, "socks5")
+        s4 = format_proxy_lines(socks4_in, "socks4")
+        combined = "\n".join([x for x in [s5, s4] if x.strip()])
+        export_path = save_proxy_export(combined)
+        return HTMLResponse(render_proxies_page(
+            message=f"Proxy-Liste gespeichert: {export_path}",
+            socks5_in=socks5_in,
+            socks4_in=socks4_in,
+            out_text=combined,
+            export_path=export_path,
+        ))
+    except Exception as e:
+        return HTMLResponse(render_proxies_page(
+            error=str(e),
+            socks5_in=socks5_in,
+            socks4_in=socks4_in,
+            out_text="",
+            export_path=PROXY_EXPORT_PATH,
+        ), status_code=400)

jd-webgui/static/style.css

@@ -15,6 +15,8 @@ th { background:#fbfbfb; text-align:left; }
 code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12px; background:#f2f2f2; padding:2px 4px; border-radius:4px; }
 .hint { color:#555; font-size: 12px; margin-top: 10px; }
 .error { color:#b00020; font-weight: 700; }
+.success { color:#1b7f3a; font-weight: 700; }
 .progress-row { display:flex; align-items:center; gap:8px; margin-top:6px; }
 .progress-text { font-size:12px; color:#333; min-width:48px; }
 .inline-form { margin-top:6px; }
+.log-area { width:100%; max-width: 920px; padding:10px; border:1px solid #ccc; border-radius:8px; background:#fff; }