返回 DeepSeek-Reasonix
anchor_safety.go
根目录 / cmd / e2ebench / anchor_safety.go
1 package main
2
3 import "fmt"
4
5 // anchorSafetySummary aggregates the content-free runtime audit. It measures
6 // where the shadow policy would differ from the legacy fresh-read policy; it
7 // cannot independently re-read source because trajectories intentionally omit
8 // paths, anchors, and line hashes.
9 type anchorSafetySummary struct {
10 Samples int `json:"samples,omitempty"`
11 ShadowAllows int `json:"shadow_allows,omitempty"`
12 LegacyAllows int `json:"legacy_allows,omitempty"`
13 ShadowOnlyAllows int `json:"shadow_only_allows,omitempty"`
14 ShadowOnlyBlocks int `json:"shadow_only_blocks,omitempty"`
15 NoEligibleReads int `json:"no_eligible_reads,omitempty"`
16 PartialWindows int `json:"partial_windows,omitempty"`
17 TargetChanged int `json:"target_changed,omitempty"`
18 NativeInvalid int `json:"native_invalid,omitempty"`
19 SameBatchReads int `json:"same_batch_reads,omitempty"`
20 MaxObservationAge int `json:"max_observation_age,omitempty"`
21 ByTaskMode map[string]int `json:"by_task_mode,omitempty"`
22 }
23
24 type anchorSafetyRecord struct {
25 Mode string `json:"mode"`
26 TaskMode string `json:"task_mode"`
27 RangeLines int `json:"range_lines"`
28 ObservationAge int `json:"observation_age"`
29 LegacyAllowed bool `json:"legacy_allowed"`
30 ShadowAllowed bool `json:"shadow_allowed"`
31 Reason string `json:"reason"`
32 SameBatchReadRejected bool `json:"same_batch_read_rejected"`
33 }
34
35 func (t *trajScan) recordAnchorSafetyAudit(a anchorSafetyRecord) {
36 if t.s.AnchorSafety == nil {
37 t.s.AnchorSafety = &anchorSafetySummary{ByTaskMode: map[string]int{}}
38 }
39 s := t.s.AnchorSafety
40 s.Samples++
41 if a.ShadowAllowed {
42 s.ShadowAllows++
43 }
44 if a.LegacyAllowed {
45 s.LegacyAllows++
46 }
47 if a.ShadowAllowed && !a.LegacyAllowed {
48 s.ShadowOnlyAllows++
49 }
50 if !a.ShadowAllowed && a.LegacyAllowed {
51 s.ShadowOnlyBlocks++
52 }
53 if a.SameBatchReadRejected {
54 s.SameBatchReads++
55 }
56 s.MaxObservationAge = max(s.MaxObservationAge, a.ObservationAge)
57 s.ByTaskMode[a.TaskMode]++
58 switch a.Reason {
59 case "would_block_no_eligible_read":
60 s.NoEligibleReads++
61 case "would_block_partial_window":
62 s.PartialWindows++
63 case "would_block_target_changed":
64 s.TargetChanged++
65 case "native_target_invalid":
66 s.NativeInvalid++
67 }
68 }
69
70 func renderAnchorSafety(results []result) string {
71 var total anchorSafetySummary
72 runs := 0
73 for _, r := range results {
74 if r.Trajectory == nil || r.Trajectory.AnchorSafety == nil || r.Trajectory.AnchorSafety.Samples == 0 {
75 continue
76 }
77 runs++
78 a := r.Trajectory.AnchorSafety
79 total.Samples += a.Samples
80 total.ShadowAllows += a.ShadowAllows
81 total.LegacyAllows += a.LegacyAllows
82 total.ShadowOnlyAllows += a.ShadowOnlyAllows
83 total.ShadowOnlyBlocks += a.ShadowOnlyBlocks
84 total.NoEligibleReads += a.NoEligibleReads
85 total.PartialWindows += a.PartialWindows
86 total.TargetChanged += a.TargetChanged
87 total.NativeInvalid += a.NativeInvalid
88 total.SameBatchReads += a.SameBatchReads
89 total.MaxObservationAge = max(total.MaxObservationAge, a.MaxObservationAge)
90 if total.ByTaskMode == nil {
91 total.ByTaskMode = map[string]int{}
92 }
93 for mode, n := range a.ByTaskMode {
94 total.ByTaskMode[mode] += n
95 }
96 }
97 if total.Samples == 0 {
98 return ""
99 }
100 line := fmt.Sprintf("**Anchor safety shadow** (%d runs): **samples** %d · **shadow allows** %d · **legacy allows** %d · **shadow-only allows** %d · **shadow-only blocks** %d",
101 runs, total.Samples, total.ShadowAllows, total.LegacyAllows, total.ShadowOnlyAllows, total.ShadowOnlyBlocks)
102 line += fmt.Sprintf(" · **same-batch reads** %d · **partial windows** %d · **target changed** %d · **native invalid** %d · **max observation age** %d",
103 total.SameBatchReads, total.PartialWindows, total.TargetChanged, total.NativeInvalid, total.MaxObservationAge)
104 if len(total.ByTaskMode) > 0 {
105 line += fmt.Sprintf(" · **interactive** %d · **loop** %d", total.ByTaskMode["interactive"], total.ByTaskMode["loop"])
106 }
107 return line + "\n\n"
108 }
109
109 lines GO