返回 last30days-skill
test_onboarding_contract.py
根目录 / tests / test_onboarding_contract.py
1 """Contract tests for the restored first-run NUX wizard in SKILL.md.
2
3 Step 0 has three branches: a **Claude Code Modal Flow** (AskUserQuestion-driven,
4 the restored v3.0.0 NUX), a **Non-Modal Prose Flow** for hosts without modals
5 (OpenClaw, Codex, Cursor, Gemini CLI), and a **Grok Bot Prose Flow** (the X
6 connector lane first, keys written only through the engine, no browser-session
7 reads). These tests assert the structural guarantees of the branches, plus the cross-cutting copy rules: the hard
8 "Step 0 before Step 1" gate, Digg threaded alongside yt-dlp, the 10,000-free-calls
9 credit count, and Threads/Pinterest kept out of the onboarding offers. They read
10 SKILL.md as text - the model's runtime contract - matching
11 tests/test_runtime_preflight_contract.py.
12
13 These lock the flow against silent re-erosion (the failure mode that orphaned the
14 wizard in PR #659 and flattened it before this restoration).
15 """
16
17 import unittest
18 from pathlib import Path
19
20 from lib import setup_wizard
21
22 ROOT = Path(__file__).resolve().parents[1]
23 SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
24 AGENTS_MD = ROOT / "AGENTS.md"
25
26
27 class TestOnboardingContract(unittest.TestCase):
28 def setUp(self):
29 self.text = SKILL_MD.read_text(encoding="utf-8")
30 # Scope assertions to Step 0 so generic substrings elsewhere in the file
31 # do not satisfy ordering/presence checks.
32 start = self.text.index("## Step 0: First-Run Setup Wizard")
33 end = self.text.index("## CRITICAL: Parse User Intent", start)
34 self.step0 = self.text[start:end]
35 # Branch slices.
36 modal_start = self.step0.index("### Claude Code Modal Flow")
37 prose_start = self.step0.index("### Non-Modal Prose Flow")
38 grok_start = self.step0.index("### Grok Bot Prose Flow")
39 manual_start = self.step0.index("### Manual Setup Guide")
40 self.modal = self.step0[modal_start:prose_start]
41 self.prose = self.step0[prose_start:grok_start]
42 self.grok = self.step0[grok_start:manual_start]
43 self.manual = self.step0[manual_start:]
44
45 # --- Platform split + hard gate ---
46
47 def test_platform_split_present(self):
48 """Step 0 routes modal-capable hosts, prose hosts, and Grok Bot to
49 three distinct flows, in that order."""
50 self.assertIn("Platform split", self.step0)
51 self.assertIn("### Claude Code Modal Flow", self.step0)
52 self.assertIn("### Non-Modal Prose Flow", self.step0)
53 self.assertIn("### Grok Bot Prose Flow", self.step0)
54 split = self.step0[self.step0.index("Platform split"):self.step0.index("### Claude Code Modal Flow")]
55 self.assertIn("Grok Bot Prose Flow", split)
56 self.assertEqual(3, len([h for h in ("### Claude Code Modal Flow", "### Non-Modal Prose Flow", "### Grok Bot Prose Flow") if h in self.step0]))
57 self.assertLess(self.step0.index("### Non-Modal Prose Flow"), self.step0.index("### Grok Bot Prose Flow"))
58
59 def test_agents_md_names_three_step0_branches(self):
60 """AGENTS.md's onboarding rule and this contract move together."""
61 agents = AGENTS_MD.read_text(encoding="utf-8")
62 self.assertIn("Step 0 has THREE branches", agents)
63 self.assertNotIn("Step 0 has TWO branches", agents)
64 for name in ("Claude Code Modal Flow", "Non-Modal Prose Flow", "Grok Bot Prose Flow"):
65 self.assertIn(name, agents, name)
66
67 def test_grok_flow_is_prose_and_connector_first(self):
68 """The third branch has no modals, leads with the X connector, and
69 never routes through a browser-session step."""
70 self.assertNotIn("AskUserQuestion", self.grok)
71 self.assertNotIn("cookie", self.grok.lower())
72 self.assertLess(self.grok.index("search_posts_all"), self.grok.index("X_BEARER_TOKEN"))
73 self.assertIn("setup --store-key", self.grok)
74
75 def test_first_run_flows_do_not_invoke_preflight(self):
76 """Status and permission inspection are not a required first-run beat.
77 `--preflight` stays an opt-in inspector; Step 0 must not dump `.env`."""
78 self.assertNotIn("--preflight", self.modal)
79 self.assertNotIn("--preflight", self.prose)
80 self.assertNotIn("--preflight", self.grok)
81 self.assertIn("Do not run it as a required first-run step", self.step0)
82 self.assertIn("Do not print `.env` contents or credential values", self.step0)
83
84 def test_first_run_gate_defers_to_step0_credential_sources(self):
85 """The cheap SETUP_COMPLETE grep is not itself a first-run verdict."""
86 start = self.text.index("**FIRST-RUN GATE")
87 end = self.text.index("\n## Step 0: First-Run Setup Wizard")
88 gate = self.text[start:end]
89 self.assertIn("FIRST_RUN_DETECTED", gate)
90 self.assertIn("A missing `.env` alone is not a first run", gate)
91 self.assertIn("That section decides first-run from every credential source", gate)
92
93 def test_complete_does_not_treat_setup_stdout_as_source_list(self):
94 self.assertIn(
95 "Setup stdout is what this run installed, not the runtime source list",
96 self.prose,
97 )
98 self.assertIn(
99 "Setup stdout is what this run installed, not the runtime source list",
100 self.grok,
101 )
102
103 def test_hard_gate_step0_before_step1(self):
104 """The erosion-resistant gate that orphaned the wizard in #659 is restored."""
105 self.assertIn("ALWAYS execute Step 0 BEFORE Step 1", self.step0)
106
107 def test_waiting_topic_continues_after_x_decline_or_setup_skip(self):
108 """Declining optional X access must never strand the requested topic."""
109 self.assertIn("RESEARCH CONTINUATION OVERRIDE", self.step0)
110 self.assertIn("declining or skipping X must never stop", self.step0)
111 self.assertIn("immediately research it with `--no-browser-cookies`", self.modal)
112 self.assertIn("immediately research it with `--no-browser-cookies`", self.prose)
113 self.assertIn("a skip or no answer is never consent", self.step0)
114
115 def test_waiting_topic_defers_optional_prompts_and_x_retry(self):
116 self.assertIn("skip the ScrapeCreators offer", self.step0)
117 self.assertIn("Do not ask another X question in the same run", self.step0)
118 self.assertIn("Offer ONE retry only when no research topic is waiting", self.modal)
119 self.assertIn("Offer ONE retry only when no research topic is waiting", self.prose)
120
121 def test_deferred_onboarding_resumes_after_the_findings(self):
122 """Deferral is same-run only: SETUP_COMPLETE=true means later runs skip
123 Step 0, so a skip-X-with-topic run must itself resume the ScrapeCreators
124 offer after the findings or the offer is dropped forever."""
125 self.assertIn("RESUME the deferred onboarding in the SAME run", self.step0)
126 self.assertIn(
127 "this run is the only chance to make the offer", self.step0
128 )
129 # Both flows: Skip-for-now, Skip-X modal option, and the prose no-path
130 # all resume the deferred offer in the same run after the findings.
131 self.assertEqual(
132 2,
133 self.modal.count(
134 "then resume Step 4 (and Step 5 if a key is saved) in the same run"
135 ),
136 )
137 self.assertIn("then resume the deferred onboarding in the same run", self.prose)
138 # The resume never turns back into a second X consent ask.
139 self.assertIn("the resume never re-asks X/browser-cookie consent", self.step0)
140 self.assertIn("Do not re-ask cookie consent as part of the resume", self.prose)
141
142 def test_x_handle_resolution_and_plan_follow_active_sources(self):
143 self.assertIn("If `ACTIVE_SOURCES_LIST` contains `x`", self.text)
144 self.assertIn("every applicable source from `ACTIVE_SOURCES_LIST`", self.text)
145 self.assertIn("Preserve X whenever it is active", self.text)
146
147 def test_post_report_x_note_is_non_blocking(self):
148 self.assertNotIn("Just-in-time X unlock", self.text)
149 self.assertIn("Optional X omission", self.text)
150 self.assertIn("finish the useful findings first", self.text)
151
152 # --- Modal flow: the restored NUX, stages in order ---
153
154 def test_modal_flow_stage_order(self):
155 """Welcome -> setup modal -> cookie consent -> SC offer -> opt-in -> picker."""
156 anchors = [
157 "Welcome to /last30days!", # welcome pitch, embedded in the setup modal
158 "How would you like to set up?",
159 "your browser's x.com cookies", # cookie-consent modal
160 "Want to add TikTok and Instagram?", # SC offer
161 "Which ScrapeCreators sources?", # source opt-in
162 "What do you want to research first?", # topic picker
163 ]
164 idxs = [self.modal.find(a) for a in anchors]
165 for a, i in zip(anchors, idxs):
166 self.assertGreater(i, -1, f"modal flow missing stage anchor: {a!r}")
167 self.assertEqual(idxs, sorted(idxs), "modal flow stages are out of order")
168
169 def test_modal_uses_askuserquestion(self):
170 self.assertIn("AskUserQuestion", self.modal)
171
172 def test_cookie_consent_names_all_installed_clis(self):
173 """The cookie-consent modal must not frame X cookies as instead-of the CLIs,
174 and must name arXiv + Techmeme (not just 'YouTube + Digg') since auto-setup
175 installs all four regardless of the cookie choice."""
176 consent = self.modal[self.modal.find("your browser's x.com cookies"):]
177 consent = consent[: consent.find("Full Disk Access")] # bound to the consent modal
178 for cli in ("yt-dlp", "Digg", "arXiv", "Techmeme"):
179 self.assertIn(cli, consent, cli)
180 # The "skip X" option still installs the CLIs (not framed as X-or-CLIs).
181 self.assertIn("Skip X - just the CLIs", consent)
182
183 def test_github_option_advertises_auto_clipboard(self):
184 """The recommended GitHub option tells the user the code is auto-copied to
185 their clipboard, so they just paste it."""
186 self.assertIn("clipboard automatically", self.modal)
187
188 def test_modal_cookie_consent_before_setup(self):
189 consent = self.modal.find("your browser's x.com cookies")
190 setup = self.modal.find("last30days.py setup")
191 self.assertGreater(consent, -1, "no cookie-consent modal in modal flow")
192 self.assertGreater(setup, -1, "no setup invocation in modal flow")
193 self.assertLess(consent, setup, "cookie consent must precede setup in modal flow")
194
195 def test_topic_picker_skips_when_topic_supplied(self):
196 """The picker documents skipping when the user already gave a topic."""
197 self.assertIn("What do you want to research first?", self.modal)
198 self.assertIn("SKIP this picker", self.modal)
199
200 # --- Prose flow: same work, modal-free ---
201
202 def test_prose_flow_has_no_modals(self):
203 self.assertNotIn("AskUserQuestion", self.prose)
204
205 def test_prose_cookie_consent_before_setup(self):
206 consent = self.prose.find("Cookie consent")
207 setup = self.prose.find("last30days.py setup")
208 self.assertGreater(consent, -1, "no cookie-consent step in prose flow")
209 self.assertGreater(setup, -1, "no setup invocation in prose flow")
210 self.assertLess(consent, setup, "cookie consent must precede setup in prose flow")
211
212 def test_prose_decline_uses_from_browser_off(self):
213 self.assertIn("FROM_BROWSER=off", self.prose)
214
215 # --- Full Disk Access remediation (both branches) ---
216
217 def test_full_disk_access_remediation_present(self):
218 self.assertIn("Permission denied reading Cookies.binarycookies", self.modal)
219 self.assertIn("Full Disk Access", self.modal)
220 self.assertIn("Permission denied reading Cookies.binarycookies", self.prose)
221 self.assertIn("Full Disk Access", self.prose)
222
223 def test_skip_path_writes_setup_complete(self):
224 """The 'Skip for now' setup choice must write SETUP_COMPLETE or the wizard loops."""
225 skip_idx = self.modal.find("If the user picks Skip for now")
226 self.assertGreater(skip_idx, -1, "no Skip-for-now handling in modal flow")
227 # The skip branch must persist the completion flag in its own paragraph.
228 skip_para = self.modal[skip_idx:skip_idx + 400]
229 self.assertIn("SETUP_COMPLETE=true", skip_para)
230
231 # --- ScrapeCreators signup + persisted edge case ---
232
233 def test_scrapecreators_signup_present_both_branches(self):
234 self.assertIn("setup --github", self.modal)
235 self.assertIn("setup --github", self.prose)
236
237 def test_persisted_false_edge_case_documented(self):
238 self.assertIn('"persisted": false', self.step0)
239
240 # --- Digg threaded alongside yt-dlp everywhere it appears ---
241
242 def test_digg_threaded_with_ytdlp(self):
243 self.assertIn("Digg", self.modal)
244 self.assertIn("Digg", self.prose)
245 self.assertIn("Digg", self.manual)
246 # The Auto-setup modal option names every installed CLI, not just two.
247 self.assertIn("yt-dlp (YouTube), Digg, arXiv, Techmeme", self.modal)
248
249 # --- Credit count = 10,000, no conflicting numbers in onboarding ---
250
251 def test_credit_count_is_10000(self):
252 self.assertIn("10,000 free calls", self.step0)
253 self.assertNotIn("1,000 free", self.step0)
254 self.assertNotIn("1000 free credit", self.step0)
255 self.assertNotIn("1000 credits", self.step0)
256 self.assertNotIn("100 free call", self.step0)
257
258 # --- Threads/Pinterest live ONLY in the Step 5 "Everything" opt-in ---
259
260 def _modal_step5(self):
261 start = self.modal.index("**Step 5:")
262 end = self.modal.index("**Step 6:", start)
263 return self.modal[start:end]
264
265 def _modal_before_step5(self):
266 # Welcome (Step 1) through the Step 4 ScrapeCreators offer.
267 return self.modal[: self.modal.index("**Step 5:")]
268
269 def test_threads_pinterest_only_in_step5_everything(self):
270 """Threads/Pinterest are offered in the Step 5 Everything tier, and
271
272 must NOT appear in the welcome or the Step 4 offer (where they would
273 read as default-on). They are opt-in via INCLUDE_SOURCES.
274 """
275 step5 = self._modal_step5()
276 self.assertIn("Threads", step5)
277 self.assertIn("Pinterest", step5)
278 before = self._modal_before_step5()
279 self.assertNotIn("Threads", before)
280 self.assertNotIn("Pinterest", before)
281
282 def test_offer_copy_names_comments_and_auto_enrichment(self):
283 """The Step 4 offer states comments are part of the default value and
284 describes the key's real Reddit/YouTube roles (empty-path Reddit
285 search backfill + yt-dlp transcript backstop) — not rate-limit
286 escalation or SC Reddit comment enrichment on the free path."""
287 before = self._modal_before_step5()
288 self.assertIn("comments", before.lower())
289 self.assertIn("Reddit", before)
290 self.assertIn("YouTube", before)
291 self.assertIn("10,000 free calls", before)
292 # Empty-only search backup (not transport/rate-limit escalation).
293 self.assertIn("returns no items", before)
294 self.assertNotIn("when they hit rate limits", before)
295 # Free-path comments are shreddit; do not claim SC comment preference.
296 self.assertNotIn("prefers ScrapeCreators for Reddit", before)
297 self.assertNotIn("enriches Reddit comments", before)
298
299 def test_step5_does_not_claim_merged_reddit_auto_enrichment(self):
300 """Step 5 must not contradict Step 4 with 'public + ScrapeCreators' merge."""
301 step5 = self._modal_step5()
302 self.assertNotIn("public + ScrapeCreators", step5)
303 self.assertNotIn("Reddit auto-enrichment", step5)
304 self.assertIn("empty-only", step5)
305
306 def test_recommended_tier_writes_comments_by_default(self):
307 """Comments are the DEFAULT: the recommended option enables YouTube +
308 TikTok + Instagram comments (posts on -> comments on)."""
309 step5 = self._modal_step5()
310 self.assertIn(
311 "INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments",
312 step5,
313 )
314 # There is no posts-only tier.
315 self.assertIn("recommended", step5.lower())
316 self.assertIn("comments", step5.lower())
317
318 def test_everything_tier_writes_full_include_sources(self):
319 """The Everything option persists the full list incl. Threads + Pinterest."""
320 step5 = self._modal_step5()
321 self.assertIn(
322 "INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments,threads,pinterest",
323 step5,
324 )
325
326 # --- Chrome-first cookie scan (U2/U3) ---
327
328 def test_cookie_consent_leads_with_chrome(self):
329 """Both flows tell the user Chrome is checked first, with the Keychain cue."""
330 for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
331 self.assertIn("Chrome", slice_text, f"{slice_name} cookie copy omits Chrome")
332 self.assertIn("Always Allow", slice_text, f"{slice_name} omits the Keychain cue")
333
334 def test_fda_reframed_as_safari_fallback(self):
335 """Full Disk Access is framed as Safari-only, not the default path."""
336 self.assertNotIn("scan your browser (Firefox/Safari)", self.modal)
337
338 def test_welcome_embedded_in_modal(self):
339 """The welcome pitch lives INSIDE the setup modal (the only always-visible
340 surface), not as a separate message/command that Claude Code folds away.
341 The engine --welcome command is kept for the non-modal prose flow."""
342 # Pitch is in the modal question.
343 self.assertIn("Welcome to /last30days!", self.modal)
344 self.assertIn("How would you like to set up?", self.modal)
345 # The modal flow explicitly does NOT run a separate --welcome command.
346 self.assertIn("Do NOT run a separate `--welcome`", self.modal)
347 # The non-modal flow still uses the engine welcome command.
348 self.assertIn("last30days.py --welcome", self.prose)
349
350 def test_stocktwits_surfaced_as_conditional(self):
351 """StockTwits is advertised in the engine welcome as a ticker/crypto-gated
352 source (welcome text moved out of SKILL.md into the engine)."""
353 self.assertIn("StockTwits", setup_wizard.render_welcome())
354
355 # --- Honest GitHub device-code copy (U4/U7) ---
356
357 def test_no_false_instant_gh_promise(self):
358 """The '~2 seconds - no browser' claim (a nonexistent code path) is gone."""
359 self.assertNotIn("~2 seconds - no browser", self.step0)
360 self.assertNotIn("Registers via GitHub CLI in ~2 seconds", self.step0)
361
362 def test_device_code_surfacing_orchestration_present(self):
363 """Both flows use the deterministic two-command split (start returns the
364 code fast, then poll) instead of a background-and-surface spinner."""
365 self.assertIn("setup --github-start", self.modal)
366 self.assertIn("setup --github-poll", self.modal)
367 self.assertIn("setup --github-start", self.prose)
368 self.assertIn("setup --github-poll", self.prose)
369
370 def test_already_registered_status_handled(self):
371 self.assertIn("already_registered", self.modal)
372 self.assertIn("already_registered", self.prose)
373
374 # --- Welcome must render before the modal (U1) ---
375
376 def test_welcome_pitch_is_in_the_modal_question(self):
377 """The welcome pitch names the core sources inside the modal question, so
378 the user sees it without expanding folded tool output. The old skip-prone
379 'IMMEDIATELY call AskUserQuestion' wording stays gone."""
380 # Pitch names the core sources right in the modal.
381 for source in ("Reddit", "X,", "YouTube", "TikTok"):
382 self.assertIn(source, self.modal, source)
383 self.assertNotIn("Then IMMEDIATELY call AskUserQuestion", self.modal)
384
385 # --- Device code surfaced with a clipboard-paste hint (U3) ---
386
387 def test_device_code_clipboard_paste_instruction(self):
388 """The GitHub flow tells the user the code is on their clipboard to paste,
389
390 and makes surfacing the code a required step (the bug the user hit).
391 """
392 self.assertIn("on your clipboard", self.modal)
393 # Surfacing the code is a required, explicit step in the new split flow.
394 self.assertIn("SHOW THE CODE", self.modal)
395 self.assertIn("just paste", self.modal)
396
397 # --- Honest 'authorized but no key' branch, distinct from auth-failed (U4) ---
398
399 def test_authorized_but_no_key_branch_present(self):
400 """A key-fetch failure after successful auth is handled honestly (likely
401
402 an already-linked account), not lumped into 'auth didn't complete'.
403 """
404 for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
405 self.assertIn("Authorized but failed to fetch API key", slice_text, slice_name)
406 self.assertIn("already linked", slice_text, slice_name)
407
408 def test_upstream_profile_error_branch_distinct_from_already_linked(self):
409 """A ScrapeCreators /profile 5xx must not be diagnosed as already-linked (#882)."""
410 for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
411 self.assertIn("ScrapeCreators profile failed", slice_text, slice_name)
412 self.assertIn("upstream_error", slice_text, slice_name)
413 self.assertIn("server error", slice_text, slice_name)
414 # Guidance forbids the already-linked misdiagnosis on this path.
415 self.assertIn("do **NOT** say", slice_text, slice_name)
416 self.assertIn("already linked", slice_text, slice_name)
417
418 # --- Legacy guarantees retained ---
419
420 def test_old_silent_wizard_instruction_removed(self):
421 self.assertNotIn("Follow the wizard's prompts end-to-end", self.text)
422
423 def test_consent_is_conversational_contract_documented(self):
424 self.assertIn("Named onboarding contract", self.step0)
425 self.assertIn("non-interactive subprocess", self.step0)
426
427
428 if __name__ == "__main__":
429 unittest.main()
430
430 lines PYTHON