| 1 | #!/bin/bash |
| 2 | set -euo pipefail |
| 3 | |
| 4 | # === V1 vs V2 Skill Test Harness === |
| 5 | # Runs all 17 test queries through both v1 and v2 SKILL.md |
| 6 | # using `claude --print` to capture real end-to-end output. |
| 7 | |
| 8 | SKILL_DIR="$HOME/.claude/skills/last30days" |
| 9 | REPO_DIR="${REPO_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" |
| 10 | CLAUDE="${CLAUDE:-$(command -v claude || echo claude)}" |
| 11 | |
| 12 | # Safety: always restore V2 SKILL.md on exit/crash |
| 13 | cleanup() { |
| 14 | if [ -f "$SKILL_DIR/SKILL.md.v2.bak" ]; then |
| 15 | echo "" |
| 16 | echo "⚠️ Restoring V2 SKILL.md from backup (script interrupted)..." |
| 17 | cp "$SKILL_DIR/SKILL.md.v2.bak" "$SKILL_DIR/SKILL.md" |
| 18 | rm -f "$SKILL_DIR/SKILL.md.v2.bak" |
| 19 | echo " ✅ V2 restored" |
| 20 | fi |
| 21 | } |
| 22 | trap cleanup EXIT |
| 23 | TIMESTAMP=$(date +%Y%m%d-%H%M%S) |
| 24 | OUT_DIR="$REPO_DIR/docs/test-results/v1-vs-v2-${TIMESTAMP}" |
| 25 | V1_DIR="$OUT_DIR/v1" |
| 26 | V2_DIR="$OUT_DIR/v2" |
| 27 | |
| 28 | mkdir -p "$V1_DIR" "$V2_DIR" |
| 29 | |
| 30 | echo "📁 Output directory: $OUT_DIR" |
| 31 | echo "" |
| 32 | |
| 33 | # All 17 test queries |
| 34 | QUERIES=( |
| 35 | "prompting techniques for chatgpt for legal questions" |
| 36 | "best clawdbot use cases" |
| 37 | "how to best setup clawdbot" |
| 38 | "prompting tips for nano banana pro for ios designs" |
| 39 | "top claude code skills" |
| 40 | "using ChatGPT to make images of dogs" |
| 41 | "research best practices for beautiful remotion animation videos in claude code" |
| 42 | "photorealistic people in nano banana pro" |
| 43 | "What are the best rap songs lately" |
| 44 | "what are people saying about DeepSeek R1" |
| 45 | "best practices for cursor rules files for Cursor" |
| 46 | "prompt advice for using suno to make killer songs in simple mode" |
| 47 | "how do I use Codex with Claude Code on same app to make it better" |
| 48 | "kanye west" |
| 49 | "howie.ai" |
| 50 | "open claw" |
| 51 | "nano banana pro prompting" |
| 52 | ) |
| 53 | |
| 54 | TYPES=( |
| 55 | "PROMPTING+TOOL" |
| 56 | "RECOMMENDATIONS" |
| 57 | "HOW-TO" |
| 58 | "PROMPTING+TOOL" |
| 59 | "RECOMMENDATIONS" |
| 60 | "GENERAL" |
| 61 | "PROMPTING" |
| 62 | "PROMPTING" |
| 63 | "RECOMMENDATIONS" |
| 64 | "NEWS" |
| 65 | "PROMPTING" |
| 66 | "PROMPTING" |
| 67 | "HOW-TO" |
| 68 | "NEWS" |
| 69 | "GENERAL" |
| 70 | "GENERAL" |
| 71 | "PROMPTING" |
| 72 | ) |
| 73 | |
| 74 | slugify() { |
| 75 | echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | cut -c1-50 |
| 76 | } |
| 77 | |
| 78 | run_version() { |
| 79 | local version="$1" |
| 80 | local outdir="$2" |
| 81 | local total=${#QUERIES[@]} |
| 82 | |
| 83 | echo "" |
| 84 | echo "==========================================" |
| 85 | echo " Running $version — $total queries" |
| 86 | echo "==========================================" |
| 87 | echo "" |
| 88 | |
| 89 | for i in "${!QUERIES[@]}"; do |
| 90 | local query="${QUERIES[$i]}" |
| 91 | local type="${TYPES[$i]}" |
| 92 | local slug |
| 93 | slug=$(slugify "$query") |
| 94 | local num=$((i + 1)) |
| 95 | local outfile="$outdir/${num}-${slug}.txt" |
| 96 | local errfile="$outdir/${num}-${slug}.stderr.txt" |
| 97 | |
| 98 | echo "[$version] ($num/$total) $query [$type]" |
| 99 | |
| 100 | local start_time |
| 101 | start_time=$(date +%s) |
| 102 | |
| 103 | # Run claude --print with the skill invocation |
| 104 | # No timeout — claude --print exits on its own; kill manually if stuck |
| 105 | if "$CLAUDE" --print \ |
| 106 | "/last30days $query" \ |
| 107 | > "$outfile" 2>"$errfile"; then |
| 108 | local end_time |
| 109 | end_time=$(date +%s) |
| 110 | local duration=$((end_time - start_time)) |
| 111 | local lines |
| 112 | lines=$(wc -l < "$outfile") |
| 113 | echo " ✅ Done — ${lines} lines, ${duration}s" |
| 114 | else |
| 115 | local exit_code=$? |
| 116 | echo " ❌ Failed (exit $exit_code)" | tee -a "$outfile" |
| 117 | fi |
| 118 | |
| 119 | # Brief pause between queries to avoid rate limits |
| 120 | sleep 3 |
| 121 | done |
| 122 | } |
| 123 | |
| 124 | # === Phase 1: Test V1 === |
| 125 | echo "📦 Backing up current V2 SKILL.md..." |
| 126 | cp "$SKILL_DIR/SKILL.md" "$SKILL_DIR/SKILL.md.v2.bak" |
| 127 | |
| 128 | echo "📥 Installing V1 SKILL.md from upstream..." |
| 129 | cd "$REPO_DIR" |
| 130 | git show upstream/main:SKILL.md | sed '/^context: fork$/d; /^agent: Explore$/d; /^disable-model-invocation: true$/d' > "$SKILL_DIR/SKILL.md" |
| 131 | cp "$SKILL_DIR/SKILL.md" "$OUT_DIR/v1-SKILL.md" |
| 132 | echo " ✅ V1 installed (stripped: context:fork, agent:Explore, disable-model-invocation)" |
| 133 | |
| 134 | run_version "V1" "$V1_DIR" |
| 135 | |
| 136 | # === Phase 2: Test V2 === |
| 137 | echo "" |
| 138 | echo "📥 Restoring V2 SKILL.md..." |
| 139 | cp "$SKILL_DIR/SKILL.md.v2.bak" "$SKILL_DIR/SKILL.md" |
| 140 | cp "$SKILL_DIR/SKILL.md" "$OUT_DIR/v2-SKILL.md" |
| 141 | echo " ✅ V2 restored" |
| 142 | |
| 143 | run_version "V2" "$V2_DIR" |
| 144 | |
| 145 | # === Phase 3: Generate summary === |
| 146 | echo "" |
| 147 | echo "==========================================" |
| 148 | echo " Generating comparison summary" |
| 149 | echo "==========================================" |
| 150 | |
| 151 | SUMMARY="$OUT_DIR/comparison-summary.md" |
| 152 | |
| 153 | cat > "$SUMMARY" << EOF |
| 154 | # V1 vs V2 Comparison Results |
| 155 | |
| 156 | Generated: $(date) |
| 157 | Output directory: $OUT_DIR |
| 158 | |
| 159 | ## Output Files |
| 160 | |
| 161 | | # | Query | Type | V1 Lines | V2 Lines | V1 Time | V2 Time | |
| 162 | |---|-------|------|----------|----------|---------|---------| |
| 163 | EOF |
| 164 | |
| 165 | for i in "${!QUERIES[@]}"; do |
| 166 | query="${QUERIES[$i]}" |
| 167 | type="${TYPES[$i]}" |
| 168 | slug=$(slugify "$query") |
| 169 | num=$((i + 1)) |
| 170 | |
| 171 | v1file="$V1_DIR/${num}-${slug}.txt" |
| 172 | v2file="$V2_DIR/${num}-${slug}.txt" |
| 173 | |
| 174 | v1lines=$(wc -l < "$v1file" 2>/dev/null || echo "ERR") |
| 175 | v2lines=$(wc -l < "$v2file" 2>/dev/null || echo "ERR") |
| 176 | |
| 177 | echo "| $num | \`$query\` | $type | $v1lines | $v2lines | — | — |" >> "$SUMMARY" |
| 178 | done |
| 179 | |
| 180 | cat >> "$SUMMARY" << 'EOF' |
| 181 | |
| 182 | ## Quick Check: Key Features |
| 183 | |
| 184 | For each query, check these v2 improvements: |
| 185 | |
| 186 | - [ ] Query parsing display (`🔍 **{TOPIC}** · {QUERY_TYPE}`) |
| 187 | - [ ] Sparse citations (not every sentence) |
| 188 | - [ ] Bold topic headers in summary |
| 189 | - [ ] Emoji stats tree (`├─ 🟠 Reddit:`) |
| 190 | - [ ] Quality checklist applied to prompts |
| 191 | - [ ] Self-check (research grounding, not generic) |
| 192 | |
| 193 | ## Scoring Guide |
| 194 | |
| 195 | Use the full scoring rubric from: |
| 196 | `docs/plans/2026-02-06-test-v1-vs-v2-comparison-plan.md` |
| 197 | |
| 198 | ## Next Step |
| 199 | |
| 200 | Have Claude read all 34 output files and generate scored comparison: |
| 201 | ``` |
| 202 | Read all files in docs/test-results/v1-vs-v2-*/v1/ and v2/ |
| 203 | Score each on the 7 dimensions from the test plan |
| 204 | Write the final analysis to docs/test-results/v1-vs-v2-*/analysis.md |
| 205 | ``` |
| 206 | EOF |
| 207 | |
| 208 | # Cleanup backup |
| 209 | rm -f "$SKILL_DIR/SKILL.md.v2.bak" |
| 210 | |
| 211 | echo "" |
| 212 | echo "✅ All done!" |
| 213 | echo "" |
| 214 | echo "📁 Results: $OUT_DIR" |
| 215 | echo "📊 Summary: $SUMMARY" |
| 216 | echo "📄 V1 files: $V1_DIR/" |
| 217 | echo "📄 V2 files: $V2_DIR/" |
| 218 | echo "" |
| 219 | echo "To review:" |
| 220 | echo " open $OUT_DIR" |
| 221 |