Merge branch 'main' into codex/add-integration-for-chatgpt.com-xrizez

Skip non-functional proxies during selection

media-webgui/app.py

@@ -18,7 +18,7 @@ from urllib.parse import urlparse
 
 import paramiko
 from fastapi import FastAPI, Form, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 
 OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "/output").rstrip("/")
@@ -97,6 +97,7 @@ class Job:
     library: str
     proxy: str
     headers: List[str]
+    progress: float
     status: str
     message: str
 
@@ -162,7 +163,6 @@ def proxy_is_usable(proxy: str) -> bool:
         return False
 
 
-
 def format_proxy_lines(raw: str, scheme: str) -> str:
     scheme = scheme.strip().lower()
     if scheme not in {"socks5", "socks4", "http", "https"}:
@@ -234,24 +234,6 @@ refresh_proxies()
 threading.Thread(target=proxy_refresh_loop, daemon=True).start()
 
 
-def parse_header_lines(raw: str) -> List[str]:
-    headers = []
-    for line in (raw or "").splitlines():
-        s = line.strip()
-        if not s or s.startswith("#"):
-            continue
-        if ":" not in s:
-            raise ValueError(f"Invalid header line: {s}")
-        name, value = s.split(":", 1)
-        name = name.strip()
-        value = value.strip()
-        if not name or not value:
-            raise ValueError(f"Invalid header line: {s}")
-        headers.append(f"{name}: {value}")
-    return headers
-
-
-
 def parse_header_lines(raw: str) -> List[str]:
     headers = []
     for line in (raw or "").splitlines():
@@ -281,20 +263,49 @@ def pick_engine(url: str, forced: str) -> str:
     return "direct"
 
 
-def run_ytdlp(url: str, out_dir: str, fmt: str, proxy: str):
+def run_ytdlp(url: str, out_dir: str, fmt: str, proxy: str, progress_cb):
-    cmd = ["yt-dlp", "-f", fmt, "-o", f"{out_dir}/%(title)s.%(ext)s", url]
+    cmd = ["yt-dlp", "--newline", "-f", fmt, "-o", f"{out_dir}/%(title)s.%(ext)s", url]
     if proxy:
         cmd += ["--proxy", proxy]
-    subprocess.check_call(cmd)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    if not proc.stdout:
+        raise RuntimeError("yt-dlp failed to start")
+    progress_re = re.compile(r"\\[download\\]\\s+([\\d.]+)%")
+    for line in proc.stdout:
+        match = progress_re.search(line)
+        if match:
+            progress_cb(float(match.group(1)))
+    ret = proc.wait()
+    if ret != 0:
+        raise subprocess.CalledProcessError(ret, cmd)
 
 
-def run_aria2(url: str, out_dir: str, proxy: str, headers: List[str] | None = None):
+def run_aria2(url: str, out_dir: str, proxy: str, progress_cb, headers: List[str] | None = None):
-    cmd = ["aria2c", "--dir", out_dir, "--allow-overwrite=true", "--auto-file-renaming=false", url]
+    cmd = [
+        "aria2c",
+        "--dir",
+        out_dir,
+        "--allow-overwrite=true",
+        "--auto-file-renaming=false",
+        "--summary-interval=1",
+        url,
+    ]
     if proxy:
         cmd += ["--all-proxy", proxy]
     for header in headers or []:
         cmd += ["--header", header]
-    subprocess.check_call(cmd)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    if not proc.stdout:
+        raise RuntimeError("aria2c failed to start")
+    percent_re = re.compile(r"\\((\\d+)%\\)")
+    for line in proc.stdout:
+        match = percent_re.search(line)
+        if match:
+            progress_cb(float(match.group(1)))
+    ret = proc.wait()
+    if ret != 0:
+        raise subprocess.CalledProcessError(ret, cmd)
+
 
 
 def md5_file(path: str) -> str:
@@ -409,6 +420,18 @@ def worker(jobid: str):
             header_note = f"Headers={len(headers)}" if headers else "Headers=none"
             job.status = "downloading"
             job.message = f"Engine={engine} Proxy={'none' if not proxy else proxy} {header_note}"
+            job.progress = 0.0
+
+        def update_progress(value: float):
+            with lock:
+                jobs[jobid].progress = max(0.0, min(100.0, value))
+
+        if engine == "ytdlp":
+            run_ytdlp(job.url, OUTPUT_DIR, YTDLP_FORMAT, proxy, update_progress)
+        elif engine == "hoster":
+            run_aria2(job.url, OUTPUT_DIR, proxy, update_progress, headers=headers)
+        else:
+            run_aria2(job.url, OUTPUT_DIR, proxy, update_progress)
 
         if engine == "ytdlp":
             run_ytdlp(job.url, OUTPUT_DIR, YTDLP_FORMAT, proxy)
@@ -459,6 +482,7 @@ def worker(jobid: str):
 
         with lock:
             job.status = "finished"
+            job.progress = 100.0
             job.message = f"OK ({len(new_files)} file(s))"
 
     except Exception as e:
@@ -488,6 +512,9 @@ def render_downloads(error: str = "") -> str:
             f"<td>{j.engine}</td>"
             f"<td>{j.library}</td>"
             f"<td>{'none' if not j.proxy else j.proxy}</td>"
+            f"<td><div class='progress' data-jobid='{j.id}'>"
+            f"<div class='progress-bar' style='width:{j.progress:.1f}%'></div>"
+            f"<span class='progress-text'>{j.progress:.1f}%</span></div></td>"
             f"<td><b>{j.status}</b><br/><small>{j.message}</small></td>"
             f"</tr>"
         )
@@ -544,6 +571,31 @@ def render_downloads(error: str = "") -> str:
       </p>
 
       <table>
+        <thead><tr><th>JobID</th><th>URL</th><th>Engine</th><th>Library</th><th>Proxy</th><th>Progress</th><th>Status</th></tr></thead>
+        <tbody>
+          {rows if rows else "<tr><td colspan='7'><em>No jobs yet.</em></td></tr>"}
+        </tbody>
+      </table>
+      <script>
+        async function refreshProgress() {{
+          try {{
+            const res = await fetch("/jobs");
+            const data = await res.json();
+            data.forEach((job) => {{
+              const el = document.querySelector(`.progress[data-jobid='${{job.id}}']`);
+              if (!el) return;
+              const bar = el.querySelector(".progress-bar");
+              const text = el.querySelector(".progress-text");
+              const pct = Math.max(0, Math.min(100, job.progress || 0));
+              bar.style.width = pct + "%";
+              text.textContent = pct.toFixed(1) + "%";
+            }});
+          }} catch (e) {{
+            console.warn("progress refresh failed", e);
+          }}
+        }}
+        setInterval(refreshProgress, 2000);
+      </script>
         <thead><tr><th>JobID</th><th>URL</th><th>Engine</th><th>Library</th><th>Proxy</th><th>Status</th></tr></thead>
         <tbody>
           {rows if rows else "<tr><td colspan='6'><em>No jobs yet.</em></td></tr>"}
@@ -558,6 +610,19 @@ def index():
     return HTMLResponse(render_downloads())
 
 
+@app.get("/jobs", response_class=JSONResponse)
+def jobs_status():
+    with lock:
+        payload = [
+            {
+                "id": job.id,
+                "progress": job.progress,
+            }
+            for job in jobs.values()
+        ]
+    return JSONResponse(payload)
+
+
 @app.post("/submit")
 def submit(
     url: str = Form(...),
@@ -589,6 +654,7 @@ def submit(
             library=library,
             proxy=chosen_proxy,
             headers=header_lines,
+            progress=0.0,
             status="queued",
             message="queued",
         )

media-webgui/static/style.css

@@ -11,3 +11,6 @@ th { background:#fbfbfb; text-align:left; }
 code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 12px; background:#f2f2f2; padding:2px 4px; border-radius:4px; }
 .hint { color:#555; font-size: 12px; margin-top: 10px; }
 .error { color:#b00020; font-weight: 700; }
+.progress { position: relative; height: 18px; background: #eee; border-radius: 10px; overflow: hidden; min-width: 120px; }
+.progress-bar { height: 100%; background: #4b7bec; transition: width 0.4s ease; }
+.progress-text { position: absolute; inset: 0; display: flex; align-items: center; justify-content: center; font-size: 12px; color: #fff; font-weight: 600; text-shadow: 0 1px 2px rgba(0,0,0,0.35); }