Merge branch 'main' into codex/fix-jdownloader-api-package-removal-error-54zoo0

Add raw MyJDownloader API fallback for removing/canceling links

jd-webgui/app.py

@@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from myjdapi import Myjdapi
 import paramiko
 from fastapi import FastAPI, Form, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 
 # ============================================================
@@ -57,6 +57,7 @@ POLL_SECONDS = float(os.environ.get("POLL_SECONDS", "5"))
 # JDownloader writes here inside container
 JD_OUTPUT_PATH = "/output"
 PROXY_EXPORT_PATH = os.environ.get("PROXY_EXPORT_PATH", "/output/jd-proxies.jdproxies")
+LOG_BUFFER_LIMIT = int(os.environ.get("LOG_BUFFER_LIMIT", "500"))
 
 URL_RE = re.compile(r"^https?://", re.I)
 
@@ -125,6 +126,21 @@ class Job:
 
 jobs: Dict[str, Job] = {}
 lock = threading.Lock()
+log_lock = threading.Lock()
+connection_logs: List[str] = []
+
+def log_connection(message: str) -> None:
+    timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    line = f"[{timestamp}] {message}"
+    with log_lock:
+        connection_logs.append(line)
+        if len(connection_logs) > LOG_BUFFER_LIMIT:
+            excess = len(connection_logs) - LOG_BUFFER_LIMIT
+            del connection_logs[:excess]
+
+def get_connection_logs() -> str:
+    with log_lock:
+        return "\n".join(connection_logs)
 
 # ============================================================
 # Core helpers
@@ -151,6 +167,7 @@ def ensure_env():
 
 def get_device():
     jd = Myjdapi()
+    log_connection(f"MyJDownloader connect as {MYJD_EMAIL or 'unknown'}")
     jd.connect(MYJD_EMAIL, MYJD_PASSWORD)
 
     wanted = (MYJD_DEVICE or "").strip()
@@ -248,6 +265,7 @@ def ffprobe_ok(path: str) -> bool:
 def ssh_connect() -> paramiko.SSHClient:
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    log_connection(f"SSH connect {JELLYFIN_USER}@{JELLYFIN_HOST}:{JELLYFIN_PORT}")
     ssh.connect(
         hostname=JELLYFIN_HOST,
         port=JELLYFIN_PORT,
@@ -270,6 +288,7 @@ def sftp_mkdirs(sftp: paramiko.SFTPClient, remote_dir: str):
 def sftp_upload(ssh: paramiko.SSHClient, local_path: str, remote_path: str):
     sftp = ssh.open_sftp()
     try:
+        log_connection(f"SFTP upload {local_path} -> {remote_path}")
         sftp_mkdirs(sftp, os.path.dirname(remote_path))
         sftp.put(local_path, remote_path)
     finally:
@@ -278,6 +297,7 @@ def sftp_upload(ssh: paramiko.SSHClient, local_path: str, remote_path: str):
 def remote_md5sum(ssh: paramiko.SSHClient, remote_path: str) -> str:
     quoted = shlex.quote(remote_path)
     cmd = f"md5sum {quoted}"
+    log_connection(f"SSH exec {cmd}")
     stdin, stdout, stderr = ssh.exec_command(cmd, timeout=120)
     out = stdout.read().decode("utf-8", "replace").strip()
     err = stderr.read().decode("utf-8", "replace").strip()
@@ -292,6 +312,7 @@ def remote_md5sum(ssh: paramiko.SSHClient, remote_path: str) -> str:
 # ============================================================
 def _http_get_json(url: str, headers: Optional[Dict[str, str]] = None) -> Any:
     req = urllib.request.Request(url, headers=headers or {})
+    log_connection(f"HTTP GET {url} (no-proxy)")
     with NO_PROXY_OPENER.open(req, timeout=20) as r:
         return json.loads(r.read().decode("utf-8", "replace"))
 
@@ -365,32 +386,16 @@ def format_proxy_lines(raw: str, scheme: str) -> str:
 
 def fetch_proxy_list(url: str) -> str:
     req = urllib.request.Request(url)
+    log_connection(f"HTTP GET {url} (no-proxy)")
     with NO_PROXY_OPENER.open(req, timeout=20) as resp:
-        return resp.read().decode("utf-8", "replace")
+        text = resp.read().decode("utf-8", "replace")
+    if "\n" not in text and re.search(r"\s", text):
+        return re.sub(r"\s+", "\n", text.strip())
+    return text
 
 def build_jdproxies_payload(text: str) -> Dict[str, Any]:
     if not text.strip():
         raise ValueError("Keine Proxy-Einträge zum Speichern.")
-    blacklist_filter = {
-        "entries": [
-            "# Dies ist ein Kommentar",
-            "// Dies ist auch ein Kommentar",
-            "# Für jdownloader.org auskommentieren",
-            "# jdownloader.org",
-            "# unten für alle Accounts mit der ID 'test *' @ jdownloader.org auskommentieren",
-            "#test@jdownloader.org",
-            "# Kommentar unten für ein Konto mit der ID 'test' @ jdownloader.org",
-            "#test$@jdownloader.org",
-            "# Sie können Muster für Konto-ID und Host verwenden, z. B. accountPattern @ hostPattern",
-            "",
-            "my.jdownloader.org",
-            "",
-            "api.jdownloader.org",
-            "",
-            "*.jdownloader.org",
-        ],
-        "type": "BLACKLIST",
-    }
     entries: List[Dict[str, Any]] = []
     type_map = {
         "socks5": "SOCKS5",
@@ -425,7 +430,7 @@ def build_jdproxies_payload(text: str) -> Dict[str, Any]:
         if not proxy_type:
             continue
         entries.append({
-            "filter": blacklist_filter,
+            "filter": None,
             "proxy": {
                 "address": parsed.hostname,
                 "password": None,
@@ -529,6 +534,7 @@ def jellyfin_refresh_library():
         try:
             url = JELLYFIN_API_BASE + path
             req = urllib.request.Request(url, headers=headers, method="POST")
+            log_connection(f"HTTP POST {url} (no-proxy)")
             with NO_PROXY_OPENER.open(req, timeout=20) as r:
                 _ = r.read()
             return
@@ -586,6 +592,31 @@ def local_paths_from_links(links: List[Dict[str, Any]], pkg_map: Dict[Any, Dict[
             out.append(p)
     return out
 
+def call_raw_jd_api(dev, endpoints: List[str], payloads: List[Dict[str, Any]]) -> bool:
+    method_candidates = ["action", "call", "api", "request"]
+    for method_name in method_candidates:
+        method = getattr(dev, method_name, None)
+        if method is None:
+            continue
+        for endpoint in endpoints:
+            for payload in payloads:
+                try:
+                    method(endpoint, payload)
+                    return True
+                except TypeError:
+                    try:
+                        method(endpoint, params=payload)
+                        return True
+                    except TypeError:
+                        try:
+                            method(endpoint, data=payload)
+                            return True
+                        except Exception:
+                            continue
+                except Exception:
+                    continue
+    return False
+
 def try_remove_from_jd(dev, links: List[Dict[str, Any]], pkg_map: Dict[Any, Dict[str, Any]]) -> Optional[str]:
     link_ids = [l.get("uuid") for l in links if l.get("uuid") is not None]
     pkg_ids = list(pkg_map.keys())
@@ -620,6 +651,14 @@ def try_remove_from_jd(dev, links: List[Dict[str, Any]], pkg_map: Dict[Any, Dict
             except Exception:
                 continue
 
+    endpoint_candidates = [
+        "downloads/removeLinks",
+        "downloadsV2/removeLinks",
+        "downloadcontroller/removeLinks",
+    ]
+    if call_raw_jd_api(dev, endpoint_candidates, payloads):
+        return None
+
     return "JDownloader-API: Paket/Links konnten nicht entfernt werden (Wrapper-Methoden nicht vorhanden)."
 
 def try_cancel_from_jd(dev, links: List[Dict[str, Any]], pkg_map: Dict[Any, Dict[str, Any]]) -> Optional[str]:
@@ -658,6 +697,14 @@ def try_cancel_from_jd(dev, links: List[Dict[str, Any]], pkg_map: Dict[Any, Dict
             except Exception:
                 continue
 
+    endpoint_candidates = [
+        "downloads/removeLinks",
+        "downloadsV2/removeLinks",
+        "downloadcontroller/removeLinks",
+    ]
+    if call_raw_jd_api(dev, endpoint_candidates, payloads):
+        return None
+
     return "JDownloader-API: Abbrechen fehlgeschlagen (Wrapper-Methoden nicht vorhanden)."
 
 def cancel_job(dev, jobid: str) -> Optional[str]:
@@ -733,6 +780,16 @@ def worker(jobid: str):
                 time.sleep(POLL_SECONDS)
                 continue
 
+            all_demo = all(is_demo_link(l.get("name", "")) for l in links)
+            if all_demo and not is_demo_link(job.url):
+                cancel_msg = cancel_job(dev, jobid)
+                with lock:
+                    job.status = "failed"
+                    base_msg = "JDownloader lieferte das Demo-Video Big Buck Bunny statt des gewünschten Links."
+                    job.message = f"{base_msg} {cancel_msg}" if cancel_msg else base_msg
+                    job.progress = 0.0
+                return
+
             all_finished = all(bool(l.get("finished")) for l in links)
             if not all_finished:
                 progress = calculate_progress(links)
@@ -827,6 +884,14 @@ def favicon():
 def jobs_get():
     return HTMLResponse(render_job_rows())
 
+@app.get("/logs", response_class=HTMLResponse)
+def logs_get():
+    return HTMLResponse(render_logs_page())
+
+@app.get("/logs/data", response_class=PlainTextResponse)
+def logs_data():
+    return PlainTextResponse(get_connection_logs())
+
 def render_job_rows() -> str:
     rows = ""
     with lock:
@@ -938,15 +1003,50 @@ def render_nav(active: str) -> str:
         "<div style='margin: 8px 0 14px 0;'>"
         + link("Downloads", "/", "downloads")
         + link("Proxies", "/proxies", "proxies")
+        + link("Logs", "/logs", "logs")
         + "</div>"
     )
 
+def render_logs_page() -> str:
+    return f"""
+    <html>
+    <head>
+      <link rel="stylesheet" href="/static/style.css">
+      <meta charset="utf-8">
+      <title>JD → Jellyfin (Logs)</title>
+      <script>
+        async function refreshLogs() {{
+          if (document.hidden) return;
+          try {{
+            const resp = await fetch('/logs/data');
+            if (!resp.ok) return;
+            const text = await resp.text();
+            const area = document.getElementById('log-body');
+            if (area) {{
+              area.value = text;
+              area.scrollTop = area.scrollHeight;
+            }}
+          }} catch (e) {{
+          }}
+        }}
+        setInterval(refreshLogs, 2000);
+        window.addEventListener('load', refreshLogs);
+      </script>
+    </head>
+    <body>
+      <h1>JD → Jellyfin</h1>
+      {render_nav("logs")}
+      <p class="hint">Verbindungs-Debugger (Echtzeit). Letzte {LOG_BUFFER_LIMIT} Einträge.</p>
+      <textarea id="log-body" class="log-area" rows="20" readonly></textarea>
+    </body>
+    </html>
+    """
+
 def render_proxies_page(
     error: str = "",
     message: str = "",
     socks5_in: str = "",
     socks4_in: str = "",
-    http_in: str = "",
     out_text: str = "",
     export_path: str = "",
 ) -> str:
@@ -976,16 +1076,11 @@ def render_proxies_page(
           <textarea name="socks4_in" rows="6" style="width:100%; max-width:860px; padding:10px; border:1px solid #ccc; border-radius:8px;">{socks4_in}</textarea>
         </div>
 
-        <div class="row">
-          <label>HTTP (ein Proxy pro Zeile, z. B. IP:PORT)</label><br/>
-          <textarea name="http_in" rows="6" style="width:100%; max-width:860px; padding:10px; border:1px solid #ccc; border-radius:8px;">{http_in}</textarea>
-        </div>
-
         <button type="submit">In JDownloader-Format umwandeln</button>
       </form>
 
       <h2 style="margin-top:18px;">JDownloader Import-Liste</h2>
-      <p class="hint">Format: <code>socks5://IP:PORT</code>, <code>socks4://IP:PORT</code>, <code>http://IP:PORT</code>. Keine Prüfung/Validierung.</p>
+      <p class="hint">Format: <code>socks5://IP:PORT</code>, <code>socks4://IP:PORT</code>. Keine Prüfung/Validierung.</p>
 
       <div class="row">
         <textarea id="out" rows="12" readonly style="width:100%; max-width:860px; padding:10px; border:1px solid #ccc; border-radius:8px;">{out_text}</textarea>
@@ -999,7 +1094,6 @@ def render_proxies_page(
       <form method="post" action="/proxies/save">
         <textarea name="socks5_in" style="display:none;">{socks5_in}</textarea>
         <textarea name="socks4_in" style="display:none;">{socks4_in}</textarea>
-        <textarea name="http_in" style="display:none;">{http_in}</textarea>
         <button type="submit">Liste als JDProxies speichern</button>
       </form>
 
@@ -1069,18 +1163,20 @@ def cancel(jobid: str):
 @app.get("/proxies", response_class=HTMLResponse)
 def proxies_get():
     try:
-        socks5_in = fetch_proxy_list("https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt")
+        socks5_in = fetch_proxy_list(
-        socks4_in = fetch_proxy_list("https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt")
+            "https://api.proxyscrape.com/v4/free-proxy-list/get?request=displayproxies&protocol=socks5&timeout=10000&country=all&ssl=yes&anonymity=elite&skip=0&limit=2000"
+        )
+        socks4_in = fetch_proxy_list(
+            "https://api.proxyscrape.com/v4/free-proxy-list/get?request=displayproxies&protocol=socks4&timeout=10000&country=all&ssl=yes&anonymity=elite&skip=0&limit=2000"
+        )
         http_in = fetch_proxy_list("https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt")
 
         s5 = format_proxy_lines(socks5_in, "socks5")
         s4 = format_proxy_lines(socks4_in, "socks4")
-        hp = format_proxy_lines(http_in, "http")
+        combined = "\n".join([x for x in [s5, s4] if x.strip()])
-        combined = "\n".join([x for x in [s5, s4, hp] if x.strip()])
         return HTMLResponse(render_proxies_page(
             socks5_in=socks5_in,
             socks4_in=socks4_in,
-            http_in=http_in,
             out_text=combined,
             export_path=PROXY_EXPORT_PATH,
         ))
@@ -1091,18 +1187,15 @@ def proxies_get():
 def proxies_post(
     socks5_in: str = Form(""),
     socks4_in: str = Form(""),
-    http_in: str = Form(""),
 ):
     try:
         s5 = format_proxy_lines(socks5_in, "socks5")
         s4 = format_proxy_lines(socks4_in, "socks4")
-        hp = format_proxy_lines(http_in, "http")
 
-        combined = "\n".join([x for x in [s5, s4, hp] if x.strip()])
+        combined = "\n".join([x for x in [s5, s4] if x.strip()])
         return HTMLResponse(render_proxies_page(
             socks5_in=socks5_in,
             socks4_in=socks4_in,
-            http_in=http_in,
             out_text=combined,
             export_path=PROXY_EXPORT_PATH,
         ))
@@ -1111,7 +1204,6 @@ def proxies_post(
             error=str(e),
             socks5_in=socks5_in,
             socks4_in=socks4_in,
-            http_in=http_in,
             out_text="",
             export_path=PROXY_EXPORT_PATH,
         ), status_code=400)
@@ -1120,19 +1212,16 @@ def proxies_post(
 def proxies_save(
     socks5_in: str = Form(""),
     socks4_in: str = Form(""),
-    http_in: str = Form(""),
 ):
     try:
         s5 = format_proxy_lines(socks5_in, "socks5")
         s4 = format_proxy_lines(socks4_in, "socks4")
-        hp = format_proxy_lines(http_in, "http")
+        combined = "\n".join([x for x in [s5, s4] if x.strip()])
-        combined = "\n".join([x for x in [s5, s4, hp] if x.strip()])
         export_path = save_proxy_export(combined)
         return HTMLResponse(render_proxies_page(
             message=f"Proxy-Liste gespeichert: {export_path}",
             socks5_in=socks5_in,
             socks4_in=socks4_in,
-            http_in=http_in,
             out_text=combined,
             export_path=export_path,
         ))
@@ -1141,7 +1230,6 @@ def proxies_save(
             error=str(e),
             socks5_in=socks5_in,
             socks4_in=socks4_in,
-            http_in=http_in,
             out_text="",
             export_path=PROXY_EXPORT_PATH,
         ), status_code=400)

jd-webgui/static/style.css

@@ -19,3 +19,4 @@ code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; fo
 .progress-row { display:flex; align-items:center; gap:8px; margin-top:6px; }
 .progress-text { font-size:12px; color:#333; min-width:48px; }
 .inline-form { margin-top:6px; }
+.log-area { width:100%; max-width: 920px; padding:10px; border:1px solid #ccc; border-radius:8px; background:#fff; }