| 1 | """YouTube comments via yt-dlp: the free, keyless path. |
| 2 | |
| 3 | ScrapeCreators used to be the only way to get YouTube comments. yt-dlp already |
| 4 | powers YouTube search and transcripts here and can fetch comments too, so the |
| 5 | comment lane no longer needs a paid key. These tests lock in that yt-dlp is |
| 6 | preferred, that ScrapeCreators still works as a fallback, and that a missing |
| 7 | key is no longer fatal. |
| 8 | """ |
| 9 | |
| 10 | import json |
| 11 | import unittest |
| 12 | from unittest import mock |
| 13 | |
| 14 | from lib import env, youtube_yt |
| 15 | from lib.subproc import SubprocResult |
| 16 | |
| 17 | |
| 18 | def _ytdlp_payload(comments): |
| 19 | """A yt-dlp --dump-single-json blob carrying `comments`.""" |
| 20 | return json.dumps({"id": "abc123", "title": "vid", "comments": comments}) |
| 21 | |
| 22 | |
| 23 | # yt-dlp's real comment shape, as emitted by --write-comments. |
| 24 | _RAW = [ |
| 25 | { |
| 26 | "author": "@BestFlorin", |
| 27 | "text": "Trump said the Hormuz strait is open", |
| 28 | "like_count": 11, |
| 29 | "_time_text": "2 days ago", |
| 30 | }, |
| 31 | { |
| 32 | "author": "@princem4006", |
| 33 | "text": "The U.S. cannot be trusted here", |
| 34 | "like_count": 7, |
| 35 | "_time_text": "1 day ago", |
| 36 | }, |
| 37 | ] |
| 38 | |
| 39 | |
| 40 | class TestFetchViaYtdlp(unittest.TestCase): |
| 41 | def test_parses_ytdlp_comments_into_canonical_shape(self): |
| 42 | """yt-dlp's like_count/_time_text map onto the engine's likes/date.""" |
| 43 | result = SubprocResult(returncode=0, stdout=_ytdlp_payload(_RAW), stderr="") |
| 44 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 45 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=result): |
| 46 | got = youtube_yt._fetch_video_comments_ytdlp("abc123", max_comments=5) |
| 47 | |
| 48 | self.assertEqual(2, len(got)) |
| 49 | self.assertEqual( |
| 50 | { |
| 51 | "author": "@BestFlorin", |
| 52 | "text": "Trump said the Hormuz strait is open", |
| 53 | "likes": 11, |
| 54 | "date": "2 days ago", |
| 55 | }, |
| 56 | got[0], |
| 57 | ) |
| 58 | |
| 59 | def test_honors_max_comments(self): |
| 60 | result = SubprocResult(returncode=0, stdout=_ytdlp_payload(_RAW), stderr="") |
| 61 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 62 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=result): |
| 63 | got = youtube_yt._fetch_video_comments_ytdlp("abc123", max_comments=1) |
| 64 | |
| 65 | self.assertEqual(1, len(got)) |
| 66 | |
| 67 | def test_returns_empty_when_ytdlp_not_installed(self): |
| 68 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False): |
| 69 | self.assertEqual([], youtube_yt._fetch_video_comments_ytdlp("abc123")) |
| 70 | |
| 71 | def test_returns_empty_on_ytdlp_failure(self): |
| 72 | """A non-zero exit is a fetch error, not an empty comment section.""" |
| 73 | result = SubprocResult(returncode=1, stdout="", stderr="boom") |
| 74 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 75 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=result): |
| 76 | self.assertEqual([], youtube_yt._fetch_video_comments_ytdlp("abc123")) |
| 77 | |
| 78 | def test_command_requests_top_sorted_comments(self): |
| 79 | """Lock the command: top-sort and the max_comments cap must be present, |
| 80 | or a refactor could silently return arbitrary (newest) comments.""" |
| 81 | result = SubprocResult(returncode=0, stdout=_ytdlp_payload(_RAW), stderr="") |
| 82 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 83 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=result) as run: |
| 84 | youtube_yt._fetch_video_comments_ytdlp("abc123", max_comments=4) |
| 85 | |
| 86 | cmd = run.call_args.args[0] |
| 87 | joined = " ".join(cmd) |
| 88 | self.assertIn("--write-comments", cmd) |
| 89 | self.assertIn("comment_sort=top", joined) |
| 90 | self.assertIn("max_comments=4", joined) |
| 91 | self.assertIn("player_client=android", joined) |
| 92 | self.assertEqual(cmd.count("--extractor-args"), 1) |
| 93 | self.assertTrue(any("watch?v=abc123" in a for a in cmd)) |
| 94 | |
| 95 | |
| 96 | class TestBackendPreference(unittest.TestCase): |
| 97 | def test_prefers_ytdlp_and_never_calls_scrapecreators(self): |
| 98 | """The free path wins: no SC credit is spent when yt-dlp delivers.""" |
| 99 | with mock.patch.object( |
| 100 | youtube_yt, |
| 101 | "_ytdlp_comments_result", |
| 102 | return_value=([{"author": "a", "text": "t", "likes": 1, "date": ""}], True), |
| 103 | ), mock.patch.object(youtube_yt.http, "get") as sc_get: |
| 104 | got = youtube_yt._fetch_video_comments("abc123", token="sk-live", max_comments=5) |
| 105 | |
| 106 | self.assertEqual(1, len(got)) |
| 107 | sc_get.assert_not_called() |
| 108 | |
| 109 | def test_falls_back_to_scrapecreators_when_ytdlp_fails(self): |
| 110 | """SC remains the backstop when yt-dlp is missing or throttled.""" |
| 111 | sc_payload = {"comments": [{"text": "from SC", "author": {"name": "@x"}, "likes": 3}]} |
| 112 | with mock.patch.object(youtube_yt, "_ytdlp_comments_result", return_value=([], False)), \ |
| 113 | mock.patch.object(youtube_yt.http, "get", return_value=sc_payload) as sc_get: |
| 114 | got = youtube_yt._fetch_video_comments("abc123", token="sk-live", max_comments=5) |
| 115 | |
| 116 | sc_get.assert_called_once() |
| 117 | self.assertEqual("from SC", got[0]["text"]) |
| 118 | |
| 119 | def test_no_token_and_ytdlp_failure_yields_no_comments_without_calling_sc(self): |
| 120 | with mock.patch.object(youtube_yt, "_ytdlp_comments_result", return_value=([], False)), \ |
| 121 | mock.patch.object(youtube_yt.http, "get") as sc_get: |
| 122 | got = youtube_yt._fetch_video_comments("abc123", token="", max_comments=5) |
| 123 | |
| 124 | self.assertEqual([], got) |
| 125 | sc_get.assert_not_called() |
| 126 | |
| 127 | def test_no_sc_fallback_when_ytdlp_succeeds_with_zero_comments(self): |
| 128 | """A video that genuinely has no comments must not burn an SC credit. |
| 129 | yt-dlp exit 0 + empty comments is success, not a throttle to retry.""" |
| 130 | ok_empty = SubprocResult(returncode=0, stdout=_ytdlp_payload([]), stderr="") |
| 131 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 132 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=ok_empty), \ |
| 133 | mock.patch.object(youtube_yt.http, "get") as sc_get: |
| 134 | got = youtube_yt._fetch_video_comments("abc123", token="sk-live", max_comments=5) |
| 135 | |
| 136 | self.assertEqual([], got) |
| 137 | sc_get.assert_not_called() |
| 138 | |
| 139 | def test_sc_fallback_fires_when_ytdlp_actually_fails(self): |
| 140 | """A non-zero exit is a real failure -> SC backstop should still fire.""" |
| 141 | failed = SubprocResult(returncode=1, stdout="", stderr="throttled") |
| 142 | sc_payload = {"comments": [{"text": "from SC", "author": {"name": "@x"}, "likes": 3}]} |
| 143 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 144 | mock.patch.object(youtube_yt.subproc, "run_with_timeout", return_value=failed), \ |
| 145 | mock.patch.object(youtube_yt.http, "get", return_value=sc_payload) as sc_get: |
| 146 | got = youtube_yt._fetch_video_comments("abc123", token="sk-live", max_comments=5) |
| 147 | |
| 148 | sc_get.assert_called_once() |
| 149 | self.assertEqual("from SC", got[0]["text"]) |
| 150 | |
| 151 | |
| 152 | class TestEnrichWithoutKey(unittest.TestCase): |
| 153 | def test_enriches_with_empty_token_when_ytdlp_available(self): |
| 154 | """A missing ScrapeCreators key must no longer disable comments.""" |
| 155 | items = [{"video_id": "abc123", "engagement": {"views": 100}}] |
| 156 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=True), \ |
| 157 | mock.patch.object( |
| 158 | youtube_yt, |
| 159 | "_fetch_video_comments", |
| 160 | return_value=[{"author": "@a", "text": "hi", "likes": 2, "date": ""}], |
| 161 | ): |
| 162 | youtube_yt.enrich_with_comments(items, token="") |
| 163 | |
| 164 | self.assertEqual("hi", items[0]["top_comments"][0]["text"]) |
| 165 | |
| 166 | def test_noop_with_no_token_and_no_ytdlp(self): |
| 167 | items = [{"video_id": "abc123", "engagement": {"views": 100}}] |
| 168 | with mock.patch.object(youtube_yt, "is_ytdlp_installed", return_value=False): |
| 169 | youtube_yt.enrich_with_comments(items, token="") |
| 170 | |
| 171 | self.assertNotIn("top_comments", items[0]) |
| 172 | |
| 173 | |
| 174 | class TestAvailabilityGate(unittest.TestCase): |
| 175 | def test_available_without_sc_key_when_ytdlp_installed(self): |
| 176 | """Comments are free now, so no key and no opt-in should be required.""" |
| 177 | with mock.patch.object(env, "is_ytdlp_available", return_value=True): |
| 178 | self.assertTrue(env.is_youtube_comments_available({})) |
| 179 | |
| 180 | def test_sc_path_still_available_when_ytdlp_missing(self): |
| 181 | cfg = { |
| 182 | "SCRAPECREATORS_API_KEY": "sk-live", |
| 183 | "INCLUDE_SOURCES": "youtube_comments", |
| 184 | } |
| 185 | with mock.patch.object(env, "is_ytdlp_available", return_value=False): |
| 186 | self.assertTrue(env.is_youtube_comments_available(cfg)) |
| 187 | |
| 188 | def test_unavailable_with_no_ytdlp_and_no_key(self): |
| 189 | with mock.patch.object(env, "is_ytdlp_available", return_value=False): |
| 190 | self.assertFalse(env.is_youtube_comments_available({})) |
| 191 | |
| 192 | def test_exclude_sources_still_suppresses_the_free_path(self): |
| 193 | """Comments going default-on must not defeat the documented off-switch.""" |
| 194 | cfg = {"EXCLUDE_SOURCES": "youtube_comments"} |
| 195 | with mock.patch.object(env, "is_ytdlp_available", return_value=True): |
| 196 | self.assertFalse(env.is_youtube_comments_available(cfg)) |
| 197 | |
| 198 | |
| 199 | if __name__ == "__main__": |
| 200 | unittest.main() |
| 201 |