返回 douyin-downloader
test_transcript_manager_audio.py
根目录 / tests / test_transcript_manager_audio.py
1 """Tests for the audio-extraction wiring inside
2 :class:`core.transcript_manager.TranscriptManager`.
3
4 Complements ``tests/test_transcript_manager.py`` (which predates this
5 spec and exercises the older paths). New behaviour covered here:
6
7 - Pre-upload audio extraction (the success path).
8 - Hard-fail on extraction error: no OpenAI call, DB recorded, return
9 dict carries ``reason="audio_extract_failed"``.
10 - Source-audio passthrough (``.m4a`` / ``.mp3`` / etc.).
11 - ``upload_audio_only=False`` legacy fallback.
12 - Tempdir cleanup is best-effort (failure logs WARNING, doesn't change
13 the task status).
14 - ``_resolve_api_key`` priority (env → settings → none).
15 - Property 2 (api_key never appears in logs).
16 """
17 from __future__ import annotations
18
19 import logging
20 from pathlib import Path
21 from typing import Any, Dict, List, Optional
22 from unittest.mock import AsyncMock
23
24 import pytest
25
26 from config import ConfigLoader
27 from core import transcript_manager as tm_mod
28 from core.audio_extraction import (
29 AudioExtractError,
30 FfmpegLocator,
31 FfmpegNonZeroExit,
32 )
33 from core.transcript_manager import TranscriptManager
34 from storage import FileManager
35
36 # ---------------------------------------------------------------------------
37 # Test doubles
38 # ---------------------------------------------------------------------------
39
40
41 class _FakeDatabase:
42 """Captures upserts so tests can assert what was written."""
43
44 def __init__(self) -> None:
45 self.transcript_jobs: List[Dict[str, Any]] = []
46
47 async def upsert_transcript_job(self, payload: Dict[str, Any]) -> None:
48 self.transcript_jobs.append(payload)
49
50
51 def _build_manager(
52 tmp_path: Path,
53 *,
54 transcript: Optional[Dict[str, Any]] = None,
55 api_key_env_value: Optional[str] = None,
56 monkeypatch: Optional[pytest.MonkeyPatch] = None,
57 ) -> tuple[TranscriptManager, _FakeDatabase, Path]:
58 """Build a TranscriptManager with a sandboxed FileManager + DB."""
59 download_root = tmp_path / "Downloaded"
60 download_root.mkdir(parents=True, exist_ok=True)
61
62 config = ConfigLoader(None)
63 config.update(
64 path=str(download_root),
65 transcript=transcript
66 or {
67 "enabled": True,
68 "output_dir": "",
69 "model": "gpt-4o-mini-transcribe",
70 "api_url": "https://api.openai.com/v1/audio/transcriptions",
71 "api_key_env": "OPENAI_API_KEY",
72 "api_key": "",
73 "upload_audio_only": True,
74 },
75 )
76
77 if api_key_env_value is not None and monkeypatch is not None:
78 monkeypatch.setenv("OPENAI_API_KEY", api_key_env_value)
79 elif monkeypatch is not None:
80 monkeypatch.delenv("OPENAI_API_KEY", raising=False)
81
82 file_manager = FileManager(str(download_root))
83 database = _FakeDatabase()
84 manager = TranscriptManager(config, file_manager, database=database)
85 return manager, database, download_root
86
87
88 @pytest.fixture(autouse=True)
89 def _reset_locator() -> None:
90 """Each test gets a fresh FfmpegLocator singleton (so an earlier test
91 that probed for unavailable ffmpeg can't poison later tests)."""
92 FfmpegLocator.reset_for_tests()
93 yield
94 FfmpegLocator.reset_for_tests()
95
96
97 def _make_video(root: Path, name: str = "demo.mp4") -> Path:
98 """Create a fake video file inside the FileManager's base path."""
99 video_path = root / "author" / "post" / name
100 video_path.parent.mkdir(parents=True, exist_ok=True)
101 video_path.write_bytes(b"\x00" * 16)
102 return video_path
103
104
105 # ---------------------------------------------------------------------------
106 # Property 5: env > settings > none
107 # ---------------------------------------------------------------------------
108
109
110 @pytest.mark.parametrize(
111 "env_val,settings_val,expected",
112 [
113 ("env-key", "settings-key", "env-key"),
114 ("", "settings-key", "settings-key"),
115 (" ", "settings-key", "settings-key"), # whitespace = empty
116 ("env-key", "", "env-key"),
117 ("", "", ""),
118 ],
119 )
120 def test_resolve_api_key_priority(
121 tmp_path: Path,
122 monkeypatch: pytest.MonkeyPatch,
123 env_val: str,
124 settings_val: str,
125 expected: str,
126 ) -> None:
127 monkeypatch.setenv("OPENAI_API_KEY", env_val)
128 config = ConfigLoader(None)
129 config.update(
130 transcript={
131 "enabled": True,
132 "api_key_env": "OPENAI_API_KEY",
133 "api_key": settings_val,
134 }
135 )
136 file_manager = FileManager(str(tmp_path / "Downloaded"))
137 manager = TranscriptManager(config, file_manager, database=None)
138 assert manager._resolve_api_key() == expected
139
140
141 # ---------------------------------------------------------------------------
142 # Successful audio extraction + upload
143 # ---------------------------------------------------------------------------
144
145
146 async def test_process_video_extracts_audio_and_uploads_mp3(
147 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
148 ) -> None:
149 manager, db, download_root = _build_manager(
150 tmp_path, api_key_env_value="sk-DEADBEEF12345678", monkeypatch=monkeypatch
151 )
152 video = _make_video(download_root)
153
154 # Stub extract_audio to return a fake mp3 in a tmp dir.
155 async def fake_extract(video_path: Path, out_dir: Path, **kwargs):
156 out_dir.mkdir(parents=True, exist_ok=True)
157 mp3 = out_dir / f"{video_path.stem}.mp3"
158 mp3.write_bytes(b"\xff\xfb\x00fake-mp3-bytes")
159 return mp3
160
161 captured: Dict[str, Any] = {}
162
163 async def fake_call(
164 *,
165 api_key: str,
166 file_path: Path,
167 filename: str,
168 content_type: str,
169 model: str,
170 ) -> Dict[str, Any]:
171 captured.update(
172 api_key=api_key,
173 file_path=file_path,
174 filename=filename,
175 content_type=content_type,
176 model=model,
177 uploaded_size=file_path.stat().st_size,
178 )
179 return {"text": "hello world"}
180
181 monkeypatch.setattr(tm_mod, "extract_audio", fake_extract)
182 monkeypatch.setattr(
183 manager, "_call_openai_transcription", fake_call
184 )
185
186 result = await manager.process_video(video, aweme_id="aw1")
187
188 assert result["status"] == "success"
189 # We uploaded the mp3, NOT the source video.
190 assert captured["filename"] == "demo.mp3"
191 assert captured["content_type"] == "audio/mpeg"
192 assert captured["uploaded_size"] == len(b"\xff\xfb\x00fake-mp3-bytes")
193 # Outputs were written to disk.
194 assert Path(result["text_path"]).read_text() == "hello world"
195 # DB shows success.
196 assert db.transcript_jobs[-1]["status"] == "success"
197
198
199 # ---------------------------------------------------------------------------
200 # Property 3: extraction failure ⇒ no fallback ⇒ no OpenAI call
201 # ---------------------------------------------------------------------------
202
203
204 async def test_process_video_audio_extract_failure_does_not_call_openai(
205 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
206 ) -> None:
207 manager, db, download_root = _build_manager(
208 tmp_path, api_key_env_value="sk-test123", monkeypatch=monkeypatch
209 )
210 video = _make_video(download_root)
211
212 async def failing_extract(video_path: Path, out_dir: Path, **kwargs):
213 raise FfmpegNonZeroExit("exit=1; stderr_tail='boom'")
214
215 open_mock = AsyncMock()
216 monkeypatch.setattr(tm_mod, "extract_audio", failing_extract)
217 monkeypatch.setattr(
218 manager, "_call_openai_transcription", open_mock
219 )
220
221 result = await manager.process_video(video, aweme_id="aw_fail")
222
223 assert result["status"] == "failed"
224 assert result["reason"] == "audio_extract_failed"
225 err_msg = result["error"]
226 assert err_msg.startswith("audio_extract_failed: nonzero_exit_code")
227 assert "exit=1" in err_msg
228
229 # OpenAI must not have been called.
230 open_mock.assert_not_called()
231
232 # DB has the same error_message.
233 assert db.transcript_jobs[-1]["status"] == "failed"
234 assert db.transcript_jobs[-1]["error_message"] == err_msg
235 assert db.transcript_jobs[-1]["skip_reason"] is None
236
237
238 async def test_process_video_audio_extract_error_classes(
239 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
240 ) -> None:
241 """All AudioExtractError subclasses end up classified the same way."""
242 manager, db, download_root = _build_manager(
243 tmp_path, api_key_env_value="sk-test", monkeypatch=monkeypatch
244 )
245 video = _make_video(download_root)
246
247 class _CustomError(AudioExtractError):
248 cause = "custom_cause"
249
250 async def failing_extract(video_path: Path, out_dir: Path, **kwargs):
251 raise _CustomError("trace_id=abc")
252
253 monkeypatch.setattr(tm_mod, "extract_audio", failing_extract)
254 open_mock = AsyncMock()
255 monkeypatch.setattr(
256 manager, "_call_openai_transcription", open_mock
257 )
258
259 result = await manager.process_video(video, aweme_id="aw_x")
260 assert result["reason"] == "audio_extract_failed"
261 assert "audio_extract_failed: custom_cause: trace_id=abc" in result["error"]
262 open_mock.assert_not_called()
263
264
265 # ---------------------------------------------------------------------------
266 # Source-audio passthrough
267 # ---------------------------------------------------------------------------
268
269
270 @pytest.mark.parametrize(
271 "extension,expected_content_type",
272 [
273 (".m4a", "audio/mp4"),
274 (".mp3", "audio/mpeg"),
275 (".wav", "audio/wav"),
276 (".aac", "audio/aac"),
277 (".opus", "audio/ogg"),
278 (".flac", "audio/flac"),
279 (".ogg", "audio/ogg"),
280 ],
281 )
282 async def test_process_video_source_audio_passthrough(
283 tmp_path: Path,
284 monkeypatch: pytest.MonkeyPatch,
285 extension: str,
286 expected_content_type: str,
287 ) -> None:
288 """When source is already audio, we skip extraction and upload as-is."""
289 manager, db, download_root = _build_manager(
290 tmp_path, api_key_env_value="sk-test", monkeypatch=monkeypatch
291 )
292 audio_path = _make_video(download_root, name=f"clip{extension}")
293
294 extract_mock = AsyncMock()
295 monkeypatch.setattr(tm_mod, "extract_audio", extract_mock)
296
297 captured: Dict[str, Any] = {}
298
299 async def fake_call(*, api_key, file_path, filename, content_type, model):
300 captured.update(
301 file_path=file_path,
302 filename=filename,
303 content_type=content_type,
304 )
305 return {"text": "ok"}
306
307 monkeypatch.setattr(manager, "_call_openai_transcription", fake_call)
308
309 result = await manager.process_video(audio_path, aweme_id="aw_audio")
310
311 assert result["status"] == "success"
312 extract_mock.assert_not_called()
313 assert captured["file_path"] == audio_path
314 assert captured["filename"] == audio_path.name
315 assert captured["content_type"] == expected_content_type
316
317
318 # ---------------------------------------------------------------------------
319 # upload_audio_only=False legacy path
320 # ---------------------------------------------------------------------------
321
322
323 async def test_process_video_legacy_upload_when_flag_disabled(
324 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
325 ) -> None:
326 manager, db, download_root = _build_manager(
327 tmp_path,
328 transcript={
329 "enabled": True,
330 "output_dir": "",
331 "model": "whisper-1",
332 "api_url": "https://api.openai.com/v1/audio/transcriptions",
333 "api_key_env": "OPENAI_API_KEY",
334 "api_key": "",
335 "upload_audio_only": False, # ← legacy
336 },
337 api_key_env_value="sk-test",
338 monkeypatch=monkeypatch,
339 )
340 video = _make_video(download_root)
341
342 extract_mock = AsyncMock()
343 monkeypatch.setattr(tm_mod, "extract_audio", extract_mock)
344
345 captured: Dict[str, Any] = {}
346
347 async def fake_call(*, api_key, file_path, filename, content_type, model):
348 captured.update(
349 file_path=file_path, filename=filename, content_type=content_type
350 )
351 return {"text": "ok"}
352
353 monkeypatch.setattr(manager, "_call_openai_transcription", fake_call)
354
355 result = await manager.process_video(video, aweme_id="aw_legacy")
356 assert result["status"] == "success"
357 extract_mock.assert_not_called()
358 assert captured["file_path"] == video
359 assert captured["filename"] == video.name
360 assert captured["content_type"] == "video/mp4"
361
362
363 # ---------------------------------------------------------------------------
364 # Tempdir cleanup is best-effort
365 # ---------------------------------------------------------------------------
366
367
368 async def test_process_video_tempdir_cleanup_failure_only_warns(
369 tmp_path: Path,
370 monkeypatch: pytest.MonkeyPatch,
371 caplog: pytest.LogCaptureFixture,
372 ) -> None:
373 manager, db, download_root = _build_manager(
374 tmp_path, api_key_env_value="sk-test", monkeypatch=monkeypatch
375 )
376 video = _make_video(download_root)
377
378 async def fake_extract(video_path: Path, out_dir: Path, **kwargs):
379 out_dir.mkdir(parents=True, exist_ok=True)
380 mp3 = out_dir / f"{video_path.stem}.mp3"
381 mp3.write_bytes(b"x")
382 return mp3
383
384 async def fake_call(*, api_key, file_path, filename, content_type, model):
385 return {"text": "ok"}
386
387 monkeypatch.setattr(tm_mod, "extract_audio", fake_extract)
388 monkeypatch.setattr(manager, "_call_openai_transcription", fake_call)
389
390 # Patch TemporaryDirectory.cleanup to raise (e.g. Windows file lock).
391 real_td = tm_mod.tempfile.TemporaryDirectory
392
393 class _BrokenTD(real_td): # type: ignore[misc]
394 def cleanup(self):
395 raise OSError("simulated lock")
396
397 monkeypatch.setattr(tm_mod.tempfile, "TemporaryDirectory", _BrokenTD)
398
399 caplog.set_level(logging.WARNING, logger="TranscriptManager")
400 # The TranscriptManager logger is configured with propagate=False
401 # (see utils.logger.setup_logger), so we must attach caplog's handler
402 # directly to capture its records.
403 tm_logger = logging.getLogger("TranscriptManager")
404 tm_logger.addHandler(caplog.handler)
405 try:
406 result = await manager.process_video(video, aweme_id="aw_cleanup")
407 finally:
408 tm_logger.removeHandler(caplog.handler)
409
410 # Task itself reports success; cleanup error is a WARNING, not an
411 # error.
412 assert result["status"] == "success"
413 assert any(
414 "Failed to clean up transcript audio temp dir" in rec.message
415 for rec in caplog.records
416 )
417
418
419 # ---------------------------------------------------------------------------
420 # Property 2: api_key never appears in log messages
421 # ---------------------------------------------------------------------------
422
423
424 async def test_process_video_does_not_log_api_key_plaintext(
425 tmp_path: Path,
426 monkeypatch: pytest.MonkeyPatch,
427 caplog: pytest.LogCaptureFixture,
428 ) -> None:
429 sentinel_key = "sk-PLAINTEXT-SECRET-DO-NOT-LEAK-12345"
430 manager, db, download_root = _build_manager(
431 tmp_path, api_key_env_value=sentinel_key, monkeypatch=monkeypatch
432 )
433 video = _make_video(download_root)
434
435 async def fake_extract(video_path: Path, out_dir: Path, **kwargs):
436 out_dir.mkdir(parents=True, exist_ok=True)
437 mp3 = out_dir / f"{video_path.stem}.mp3"
438 mp3.write_bytes(b"x")
439 return mp3
440
441 async def fake_call(*, api_key, file_path, filename, content_type, model):
442 # Verify the manager DID resolve the env-var key.
443 assert api_key == sentinel_key
444 return {"text": "ok"}
445
446 monkeypatch.setattr(tm_mod, "extract_audio", fake_extract)
447 monkeypatch.setattr(manager, "_call_openai_transcription", fake_call)
448
449 caplog.set_level(logging.DEBUG)
450 # TranscriptManager logger has propagate=False (see utils.logger);
451 # attach the caplog handler directly to make sure we'd see any leak.
452 tm_logger = logging.getLogger("TranscriptManager")
453 tm_logger.addHandler(caplog.handler)
454 try:
455 await manager.process_video(video, aweme_id="aw_secret")
456 finally:
457 tm_logger.removeHandler(caplog.handler)
458
459 full_log = "\n".join(rec.getMessage() for rec in caplog.records)
460 assert sentinel_key not in full_log
461
462
463 async def test_call_openai_transcription_redacts_api_key_in_error_body(
464 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
465 ) -> None:
466 """When the upstream returns a 4xx body that echoes the bearer
467 token, the resulting RuntimeError must NOT carry the raw key —
468 otherwise it lands in ``transcript_jobs.error_message`` and a
469 later DB dump leaks it."""
470 sentinel = "sk-LEAK-PROBE-1234567890ABCDEF"
471 manager, db, download_root = _build_manager(
472 tmp_path, api_key_env_value=sentinel, monkeypatch=monkeypatch
473 )
474 video = _make_video(download_root)
475
476 # Skip extract_audio so we go straight to _call_openai_transcription.
477 async def passthrough_extract(video_path, out_dir, **kwargs):
478 out_dir.mkdir(parents=True, exist_ok=True)
479 mp3 = out_dir / f"{video_path.stem}.mp3"
480 mp3.write_bytes(b"\x00")
481 return mp3
482
483 monkeypatch.setattr(tm_mod, "extract_audio", passthrough_extract)
484
485 # Stub aiohttp's POST to return a 401 body that echoes the bearer
486 # token exactly the way some misbehaving upstreams do.
487 class FakeResp:
488 status = 401
489
490 async def text(self):
491 return f"<html>auth header was: Bearer {sentinel}</html>"
492
493 async def __aenter__(self):
494 return self
495
496 async def __aexit__(self, *exc):
497 return False
498
499 class FakePostCtx:
500 async def __aenter__(self):
501 return FakeResp()
502
503 async def __aexit__(self, *exc):
504 return False
505
506 class FakeSession:
507 def __init__(self, *a, **k):
508 pass
509
510 async def __aenter__(self):
511 return self
512
513 async def __aexit__(self, *a):
514 return False
515
516 def post(self, *a, **k):
517 return FakePostCtx()
518
519 monkeypatch.setattr("aiohttp.ClientSession", FakeSession)
520
521 result = await manager.process_video(video, aweme_id="aw_redact")
522 assert result["status"] == "failed"
523 err_msg = result["error"]
524 assert sentinel not in err_msg, (
525 f"raw api_key leaked into error_message: {err_msg!r}"
526 )
527 # Masked form should be present so the user can still tell which
528 # key was used.
529 assert "sk-L...CDEF" in err_msg
530 # DB record matches.
531 db_msg = db.transcript_jobs[-1]["error_message"]
532 assert sentinel not in db_msg
533 assert "sk-L...CDEF" in db_msg
534
535
536 # ---------------------------------------------------------------------------
537 # Missing api_key still produces the legacy skip path
538 # ---------------------------------------------------------------------------
539
540
541 async def test_process_video_missing_api_key_skips(
542 tmp_path: Path, monkeypatch: pytest.MonkeyPatch
543 ) -> None:
544 manager, db, download_root = _build_manager(
545 tmp_path, api_key_env_value=None, monkeypatch=monkeypatch
546 )
547 video = _make_video(download_root)
548
549 extract_mock = AsyncMock()
550 open_mock = AsyncMock()
551 monkeypatch.setattr(tm_mod, "extract_audio", extract_mock)
552 monkeypatch.setattr(manager, "_call_openai_transcription", open_mock)
553
554 result = await manager.process_video(video, aweme_id="aw_no_key")
555
556 assert result == {"status": "skipped", "reason": "missing_api_key"}
557 extract_mock.assert_not_called()
558 open_mock.assert_not_called()
559 assert db.transcript_jobs[-1]["skip_reason"] == "missing_api_key"
560
560 lines PYTHON