| 1 | """纯 Python 的后台下载任务模型，不依赖 FastAPI。 |
| 2 | |
| 3 | 将 job 生命周期从 HTTP 层解耦，便于被 CLI 以外的入口复用（如未来的 MCP server）。 |
| 4 | """ |
| 5 | |
| 6 | from __future__ import annotations |
| 7 | |
| 8 | import asyncio |
| 9 | import time |
| 10 | import uuid |
| 11 | from datetime import datetime, timezone |
| 12 | from typing import Any, Awaitable, Callable, Dict, List, Optional |
| 13 | |
| 14 | |
| 15 | def _now_iso() -> str: |
| 16 | # 统一使用 timezone-aware UTC ISO-8601 字符串 |
| 17 | return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") |
| 18 | |
| 19 | |
| 20 | class JobStatus: |
| 21 | PENDING = "pending" |
| 22 | RUNNING = "running" |
| 23 | SUCCESS = "success" |
| 24 | FAILED = "failed" |
| 25 | |
| 26 | TERMINAL = frozenset({SUCCESS, FAILED}) |
| 27 | |
| 28 | |
| 29 | class DownloadJob: |
| 30 | def __init__(self, job_id: str, url: str): |
| 31 | self.job_id = job_id |
| 32 | self.url = url |
| 33 | self.status = JobStatus.PENDING |
| 34 | self.created_at = _now_iso() |
| 35 | self.started_at: Optional[str] = None |
| 36 | self.finished_at: Optional[str] = None |
| 37 | # 单调时钟时间戳，用于 TTL / LRU 剪裁（不受系统时钟跳变影响） |
| 38 | self.finished_monotonic: Optional[float] = None |
| 39 | self.total = 0 |
| 40 | self.success = 0 |
| 41 | self.failed = 0 |
| 42 | self.skipped = 0 |
| 43 | self.error: Optional[str] = None |
| 44 | self._task: Optional[asyncio.Task] = None |
| 45 | |
| 46 | def to_dict(self) -> Dict[str, Any]: |
| 47 | return { |
| 48 | "job_id": self.job_id, |
| 49 | "url": self.url, |
| 50 | "status": self.status, |
| 51 | "created_at": self.created_at, |
| 52 | "started_at": self.started_at, |
| 53 | "finished_at": self.finished_at, |
| 54 | "total": self.total, |
| 55 | "success": self.success, |
| 56 | "failed": self.failed, |
| 57 | "skipped": self.skipped, |
| 58 | "error": self.error, |
| 59 | } |
| 60 | |
| 61 | |
| 62 | class JobManager: |
| 63 | """内存 job 存储 + 并发执行器，带 TTL + 容量上限。 |
| 64 | |
| 65 | 不做持久化——进程重启就丢失——因为当前目标只是暴露 HTTP 接口。 |
| 66 | 如需持久化可以后续在此加一层 SQLite。 |
| 67 | |
| 68 | 剪裁策略： |
| 69 | - 每次 submit 前先剪裁一次： |
| 70 | a. 丢弃 finished_monotonic 超过 job_ttl_seconds 的终态 job； |
| 71 | b. 若剩余总数仍超过 max_jobs，按 finished_monotonic 升序淘汰最老的终态 job； |
| 72 | c. in-flight（pending/running）job 永不淘汰。 |
| 73 | """ |
| 74 | |
| 75 | DEFAULT_MAX_JOBS = 500 |
| 76 | DEFAULT_JOB_TTL_SECONDS = 24 * 3600 # 24 小时 |
| 77 | |
| 78 | def __init__( |
| 79 | self, |
| 80 | executor: Callable[[str], Awaitable[Dict[str, int]]], |
| 81 | *, |
| 82 | max_concurrency: int = 2, |
| 83 | max_jobs: int = DEFAULT_MAX_JOBS, |
| 84 | job_ttl_seconds: float = DEFAULT_JOB_TTL_SECONDS, |
| 85 | ): |
| 86 | self.executor = executor |
| 87 | self._jobs: Dict[str, DownloadJob] = {} |
| 88 | self._semaphore = asyncio.Semaphore(max(1, max_concurrency)) |
| 89 | self._lock = asyncio.Lock() |
| 90 | self.max_jobs = max(1, int(max_jobs)) |
| 91 | self.job_ttl_seconds = max(0.0, float(job_ttl_seconds)) |
| 92 | |
| 93 | async def submit(self, url: str) -> DownloadJob: |
| 94 | job_id = uuid.uuid4().hex[:12] |
| 95 | job = DownloadJob(job_id=job_id, url=url) |
| 96 | async with self._lock: |
| 97 | self._prune_locked() |
| 98 | self._jobs[job_id] = job |
| 99 | # 异步调度，立即返回 job 给调用方 |
| 100 | job._task = asyncio.create_task(self._run(job)) |
| 101 | return job |
| 102 | |
| 103 | def _prune_locked(self) -> None: |
| 104 | """持锁内调用：按 TTL + 容量上限剪裁终态 job。""" |
| 105 | now = time.monotonic() |
| 106 | |
| 107 | # 1) TTL |
| 108 | if self.job_ttl_seconds > 0: |
| 109 | expired_ids = [ |
| 110 | jid |
| 111 | for jid, j in self._jobs.items() |
| 112 | if j.status in JobStatus.TERMINAL |
| 113 | and j.finished_monotonic is not None |
| 114 | and (now - j.finished_monotonic) > self.job_ttl_seconds |
| 115 | ] |
| 116 | for jid in expired_ids: |
| 117 | self._jobs.pop(jid, None) |
| 118 | |
| 119 | # 2) 容量上限：只淘汰终态 job，保留 in-flight |
| 120 | if len(self._jobs) < self.max_jobs: |
| 121 | return |
| 122 | terminal_jobs = [ |
| 123 | (j.finished_monotonic or 0.0, jid) |
| 124 | for jid, j in self._jobs.items() |
| 125 | if j.status in JobStatus.TERMINAL |
| 126 | ] |
| 127 | terminal_jobs.sort(key=lambda pair: pair[0]) |
| 128 | overflow = len(self._jobs) - self.max_jobs + 1 # +1 是为新 job 腾位 |
| 129 | for _, jid in terminal_jobs[:overflow]: |
| 130 | self._jobs.pop(jid, None) |
| 131 | |
| 132 | async def _run(self, job: DownloadJob) -> None: |
| 133 | async with self._semaphore: |
| 134 | job.status = JobStatus.RUNNING |
| 135 | job.started_at = _now_iso() |
| 136 | try: |
| 137 | counts = await self.executor(job.url) |
| 138 | job.total = int(counts.get("total", 0)) |
| 139 | job.success = int(counts.get("success", 0)) |
| 140 | job.failed = int(counts.get("failed", 0)) |
| 141 | job.skipped = int(counts.get("skipped", 0)) |
| 142 | # 只要跑完就是 success；具体成功/失败个数通过字段区分 |
| 143 | job.status = JobStatus.SUCCESS if job.failed == 0 else JobStatus.FAILED |
| 144 | except Exception as exc: |
| 145 | job.status = JobStatus.FAILED |
| 146 | job.error = f"{type(exc).__name__}: {exc}" |
| 147 | finally: |
| 148 | job.finished_at = _now_iso() |
| 149 | job.finished_monotonic = time.monotonic() |
| 150 | |
| 151 | async def get(self, job_id: str) -> Optional[DownloadJob]: |
| 152 | async with self._lock: |
| 153 | return self._jobs.get(job_id) |
| 154 | |
| 155 | async def list_jobs(self) -> List[DownloadJob]: |
| 156 | async with self._lock: |
| 157 | return list(self._jobs.values()) |
| 158 | |
| 159 | async def shutdown(self) -> None: |
| 160 | """等待所有 pending/running 任务结束。""" |
| 161 | tasks = [j._task for j in self._jobs.values() if j._task is not None] |
| 162 | if tasks: |
| 163 | await asyncio.gather(*tasks, return_exceptions=True) |
| 164 |