| 1 | """Host-contract tests for non-modal agent runtimes.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import re |
| 6 | from pathlib import Path |
| 7 | |
| 8 | ROOT = Path(__file__).resolve().parents[1] |
| 9 | SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" |
| 10 | |
| 11 | |
| 12 | def _prose_flow() -> str: |
| 13 | text = SKILL_MD.read_text(encoding="utf-8") |
| 14 | start_marker = "### Non-Modal Prose Flow" |
| 15 | # The Grok Bot Prose Flow (third Step 0 branch) sits between the prose |
| 16 | # flow and the Manual Setup Guide; slice the prose flow alone. |
| 17 | end_marker = "### Grok Bot Prose Flow" |
| 18 | start = text.find(start_marker) |
| 19 | assert start != -1, f"missing section marker: {start_marker}" |
| 20 | end = text.find(end_marker, start) |
| 21 | assert end != -1, f"missing section marker: {end_marker}" |
| 22 | return text[start:end] |
| 23 | |
| 24 | |
| 25 | def test_non_modal_hosts_are_named(): |
| 26 | prose = _prose_flow() |
| 27 | for host in ("Codex", "Cursor", "Gemini CLI", "raw CLI"): |
| 28 | assert host in prose |
| 29 | |
| 30 | |
| 31 | def test_non_modal_cookie_consent_uses_engine_allow_flag(): |
| 32 | prose = _prose_flow() |
| 33 | consent = prose.index("Cookie consent") |
| 34 | allow = prose.index("setup --allow-browser-cookies") |
| 35 | decline = prose.index("FROM_BROWSER=off") |
| 36 | assert consent < allow |
| 37 | assert consent < decline |
| 38 | |
| 39 | |
| 40 | def test_non_modal_does_not_require_preflight_before_cookie_consent(): |
| 41 | prose = _prose_flow() |
| 42 | consent = prose.index("Cookie consent") |
| 43 | assert "--preflight" not in prose |
| 44 | assert consent > prose.index("Welcome") |
| 45 | |
| 46 | |
| 47 | def test_non_modal_completion_mentions_project_trust(): |
| 48 | prose = _prose_flow() |
| 49 | assert "LAST30DAYS_TRUST_PROJECT_CONFIG=1" in prose |
| 50 | assert "Codex desktop" in prose |
| 51 | |
| 52 | |
| 53 | def _step0_search_contract() -> str: |
| 54 | text = SKILL_MD.read_text(encoding="utf-8") |
| 55 | start_marker = "**STEP 0 - RESOLVE HOST WEB SEARCH FIRST.**" |
| 56 | end_marker = "**FIRST-RUN GATE" |
| 57 | start = text.find(start_marker) |
| 58 | assert start != -1, f"missing section marker: {start_marker}" |
| 59 | end = text.find(end_marker, start) |
| 60 | assert end != -1, f"missing section marker: {end_marker}" |
| 61 | return text[start:end] |
| 62 | |
| 63 | |
| 64 | def test_host_web_search_uses_available_capability_not_specific_tool_name(): |
| 65 | step0 = _step0_search_contract() |
| 66 | assert "usable web-search tool" in step0 |
| 67 | assert "built in, exposed as a deferred tool, or provided by an installed connector" in step0 |
| 68 | assert "Brave, Firecrawl, Exa, Serper" in step0 |
| 69 | assert "If your host requires loading, selecting, or enabling the web-search tool" in step0 |
| 70 | assert "Do not fail the skill just because one particular schema lookup or tool name is unavailable" in step0 |
| 71 | |
| 72 | |
| 73 | def test_no_host_search_uses_auto_resolve_and_leaves_native_signal_unset(): |
| 74 | step0 = _step0_search_contract() |
| 75 | assert "If no web-search tool is available in the agent session" in step0 |
| 76 | assert "--auto-resolve" in step0 |
| 77 | assert "LAST30DAYS_NATIVE_SEARCH=1" in step0 |
| 78 | assert "Leave it unset when the agent session has no web-search tool" in step0 |
| 79 | |
| 80 | |
| 81 | def _law8_block() -> str: |
| 82 | text = SKILL_MD.read_text(encoding="utf-8") |
| 83 | start = text.find("**LAW 8 -") |
| 84 | assert start != -1, "missing LAW 8 marker" |
| 85 | end = text.find("**LAW 9 -", start) |
| 86 | assert end != -1, "missing LAW 9 marker (LAW 8 block end)" |
| 87 | return text[start:end] |
| 88 | |
| 89 | |
| 90 | def test_law8_is_renderer_aware_with_both_regimes(): |
| 91 | # LAW 8 must keep the inline-link default for hidden-link hosts AND carry a |
| 92 | # plain-label branch for visible-URL hosts. Codex rendered every inline link |
| 93 | # as `label (https://...)`, so a single-renderer LAW 8 produced URL soup. |
| 94 | law8 = _law8_block() |
| 95 | # Hidden-link hosts are Claude Code AND Grok Bot / Cursor agent chat: the |
| 96 | # 2026-09-01 Grok Bot brief showed Cursor agent chat hides markdown URLs |
| 97 | # like Claude Code, so lumping it with Codex produced unclickable cites. |
| 98 | assert "Hidden-link hosts (Claude Code; Grok Bot / Cursor agent chat)" in law8 |
| 99 | assert "Visible-URL hosts (Codex" in law8 |
| 100 | assert "URL soup" in law8 |
| 101 | # Hidden-link default must remain inline `[name](url)` (no Claude Code regression). |
| 102 | assert "`[name](url)`" in law8 |
| 103 | |
| 104 | |
| 105 | def test_law8_host_detection_is_deterministic_via_claudecode(): |
| 106 | law8 = _law8_block() |
| 107 | assert "CLAUDECODE" in law8 |
| 108 | # CURSOR_AGENT is the second deterministic hidden-link signal (Grok Bot / |
| 109 | # Cursor agent chat). Dropping it regresses those chats to plain labels. |
| 110 | assert "CURSOR_AGENT" in law8 |
| 111 | assert "Grok Bot" in law8 |
| 112 | # The detection must be stated as deterministic, not left to the model guessing. |
| 113 | assert "do not guess" in law8 |
| 114 | |
| 115 | |
| 116 | def test_law8_visible_url_hosts_exclude_cursor_and_split_from_step0(): |
| 117 | # Grok Bot / Cursor agent chat hides markdown URLs, so Cursor must not be |
| 118 | # named a visible-URL citation host anywhere. Cursor stays a NON-MODAL |
| 119 | # SETUP host (see test_non_modal_hosts_are_named) - the citation renderer |
| 120 | # is a different axis, and LAW 8 must not claim it is the Step 0 split. |
| 121 | text = SKILL_MD.read_text(encoding="utf-8") |
| 122 | visible_lists = re.findall(r"[Vv]isible-URL hosts? \(([^)]*)\)", text) |
| 123 | assert visible_lists, "no visible-URL host list found" |
| 124 | assert any("Codex" in hosts for hosts in visible_lists) |
| 125 | for hosts in visible_lists: |
| 126 | assert "Cursor" not in hosts, f"Cursor named as visible-URL host: {hosts!r}" |
| 127 | law8 = _law8_block() |
| 128 | assert "Gemini CLI, raw CLI" in law8 |
| 129 | assert "is the same split" not in law8 |
| 130 | |
| 131 | |
| 132 | def test_law8_wrap_list_includes_u_name_and_github_repo_first_mentions(): |
| 133 | # u/name comment authors and GitHub repos must be in the wrap-every-citation |
| 134 | # list, with URLs copied from the comment row / engine evidence block. |
| 135 | law8 = _law8_block() |
| 136 | assert "u/name" in law8 |
| 137 | assert "GitHub repo" in law8 |
| 138 | assert "never guess" in law8.lower() |
| 139 | # GitHub evidence can carry an issue/PR URL, not the repo root: the label |
| 140 | # must match what the URL opens - never `[owner/repo]` over an item URL, |
| 141 | # and never an item URL trimmed to a guessed repo root. |
| 142 | assert "a label that matches what that URL opens" in law8 |
| 143 | assert "never trim an item URL down to a guessed repo root" in law8 |
| 144 | |
| 145 | |
| 146 | def test_law8_post_synthesis_self_check_branches_on_both_env_signals(): |
| 147 | # The post-synthesis self-check is the env-branching gate; it must branch |
| 148 | # on CLAUDECODE or CURSOR_AGENT, and PRE-PRESENT is a supplemental sweep. |
| 149 | law8 = _law8_block() |
| 150 | start = law8.index("Post-synthesis self-check") |
| 151 | self_check = law8[start:] |
| 152 | assert "`CLAUDECODE` or `CURSOR_AGENT` set" in self_check |
| 153 | assert "both `CLAUDECODE` and `CURSOR_AGENT` unset" in self_check |
| 154 | assert "not a substitute" in self_check |
| 155 | |
| 156 | |
| 157 | def test_citation_renderer_host_list_is_mirrored_outside_law8(): |
| 158 | # LAW 9, FUN CONTENT, CITATION PRIORITY, and the PRE-PRESENT sweep must |
| 159 | # carry the same renderer split - Grok Bot / Cursor agent chat hidden-link, |
| 160 | # Codex/Gemini CLI/raw CLI visible-URL - so a chunked read of any one |
| 161 | # section cannot resurrect "Cursor is visible-URL". |
| 162 | text = SKILL_MD.read_text(encoding="utf-8") |
| 163 | law9_start = text.index("**LAW 9 -") |
| 164 | law9 = text[law9_start : text.index("**LAW 10 -", law9_start)] |
| 165 | assert "hidden-link host (Claude Code; Grok Bot / Cursor agent chat)" in law9 |
| 166 | fun_start = text.index("**FUN CONTENT") |
| 167 | fun = text[fun_start : fun_start + 2000] |
| 168 | assert "Grok Bot / Cursor agent chat" in fun |
| 169 | citation_start = text.index("**URL formatting is governed by LAW 8**") |
| 170 | citation = text[citation_start : citation_start + 1500] |
| 171 | assert "hidden-link hosts (Claude Code; Grok Bot / Cursor agent chat)" in citation |
| 172 | assert "Codex/Gemini CLI/raw CLI" in citation |
| 173 | pre_present_start = text.index("## PRE-PRESENT SELF-CHECK") |
| 174 | pre_present = text[pre_present_start:] |
| 175 | assert "`CLAUDECODE` or `CURSOR_AGENT` set" in pre_present |
| 176 | |
| 177 | |
| 178 | def test_plan_invocation_warns_against_bash_lc_apostrophe_wrapper(): |
| 179 | # Codex aborted its first engine run by wrapping the query-plan heredoc in |
| 180 | # `bash -lc '...'`; the outer single quote ended at the first apostrophe in a |
| 181 | # ranking string. The guidance must steer off that wrapper explicitly. |
| 182 | text = SKILL_MD.read_text(encoding="utf-8") |
| 183 | assert "bash -lc '...'" in text |
| 184 | assert "unmatched" in text |
| 185 | |
| 186 | |
| 187 | def test_step055_documents_dedicated_vs_broad_subreddits(): |
| 188 | # Step 0.55 must instruct the model to split entity-home (dedicated) subs from |
| 189 | # broad subs and pass them via --dedicated-subreddits, which the engine pulls |
| 190 | # in full and exempts from the relevance floor. |
| 191 | text = SKILL_MD.read_text(encoding="utf-8") |
| 192 | assert "RESOLVED_DEDICATED_SUBREDDITS" in text |
| 193 | assert "--dedicated-subreddits" in text |
| 194 | assert "relevance floor" in text |
| 195 |