Coverage for orchestrant/benchmark/correctness.py: 97%

1"""The speed runner's correctness probe: its prompts, their kinds, its verdict.""" 

2 

3import re 

4from typing import NamedTuple 

5 

6 

7# Integrity misses (a broken kernel's signature) decide the verdict; capability misses are the model's. 

8INTEGRITY = "integrity" 

9CAPABILITY = "capability" 

10KINDS = (INTEGRITY, CAPABILITY) 

11NO_RESULT = "NO RESULT" 

12COUNT_FIELDS = ("score", "total", "wrong", "truncated", "errors") 

13# The prompt preview ties an old report's item to its kind and is bench_compare's case key. 

14PREVIEW = 60 

15 

16 

17class Probe(NamedTuple): 

18 kind: str 

19 prompt: str 

20 accepted: list[str] 

21 

22 

23# Each entry's first accepted form is the recorded one. 

24CORRECTNESS_PROBES = [ 

25 # A SMALL multiplication: a healthy 4B could not finish 847*293 in 4000 thinking tokens. 

26 Probe(INTEGRITY, "What is 23 * 17? Reply with only the number.", ["391"]), 

27 # Trivia with a famous wrong answer: Phi-4-mini says Sydney. 

28 Probe( 

29 CAPABILITY, 

30 "What is the capital of Australia? Reply with only the city name.", 

31 ["canberra"], 

32 ), 

33 # Tokenisation: one Q4_0 file answered 5 on the GenieX CPU lane and 4 on its GPU lane. 

34 Probe( 

35 CAPABILITY, 

36 "How many times does the letter 'r' appear in the word strawberry? " 

37 "Reply with only the digit.", 

38 ["3"], 

39 ), 

40 # Trick reasoning: Llama-3.2-3B says 25. 

41 Probe( 

42 CAPABILITY, 

43 "If 5 machines make 5 widgets in 5 minutes, how many minutes do 100 " 

44 "machines need to make 100 widgets? Reply with only the number.", 

45 ["5"], 

46 ), 

47 Probe(INTEGRITY, "What is 17 squared? Reply with only the number.", ["289"]), 

48 # Trick comparison: Llama-3.2-3B and Phi-4-mini say 9.11. 

49 Probe( 

50 CAPABILITY, 

51 "Which number is larger, 9.11 or 9.9? Reply with only the number.", 

52 ["9.9"], 

53 ), 

54 # Unverified: the first --correctness run on each lane is their check. 

55 Probe(INTEGRITY, "What is 100 minus 37? Reply with only the number.", ["63"]), 

56 Probe( 

57 INTEGRITY, 

58 "What number comes next: 2, 4, 6, 8? Reply with only the number.", 

59 ["10"], 

60 ), 

61 Probe( 

62 INTEGRITY, 

63 "How many days are in one week? Reply with only the number.", 

64 ["7", "seven"], 

65 ), 

66 Probe( 

67 INTEGRITY, 

68 "Remember the word pelican. Which word did I ask you to remember? " 

69 "Reply with only the word.", 

70 ["pelican"], 

71 ), 

72] 

73 

74_KIND_BY_PREVIEW = {p.prompt[:PREVIEW]: p.kind for p in CORRECTNESS_PROBES} 

75_CUT_PREVIEW = "<truncated inside <think>, raise --correctness-max-tokens>" 

76 

77 

78def _answer_matches(content, accepted): 

79 """Does the model's FINAL answer (after </think>) contain an accepted string?""" 

80 # An unclosed <think> ran out of budget: its intermediate values are not answers. 

81 if "<think>" in content and "</think>" not in content: 

82 return False 

83 

84 answer = content.split("</think>")[-1] if "</think>" in content else content 

85 # Bias to the end: the final answer is what counts, not a mid-stream aside. 

86 answer = answer[-400:].lower().replace(",", "").replace("*", "") 

87 for exp in accepted: 

88 # A sentence-ending "." still matches, ".<digit>" does not: "3" is not in "3.5" or "13". 

89 if re.search(rf"(?<![\w.]){re.escape(exp.lower())}(?!\w)(?!\.\d)", answer): 

90 return True 

91 return False 

92 

93 

94def graded_item(probe, content): 

95 """One answered probe, as the report's `items` record it.""" 

96 truncated = "<think>" in content and "</think>" not in content 

97 answer = content.split("</think>")[-1].strip() 

98 return { 

99 "prompt": probe.prompt[:PREVIEW], 

100 "kind": probe.kind, 

101 "expected": probe.accepted[0], 

102 "answer_preview": _CUT_PREVIEW if truncated else answer[:80], 

103 "truncated": truncated, 

104 "correct": _answer_matches(content, probe.accepted), 

105 } 

106 

107 

108def errored_item(probe, error): 

109 """One probe the endpoint never answered.""" 

110 return { 

111 "prompt": probe.prompt[:PREVIEW], 

112 "kind": probe.kind, 

113 "expected": probe.accepted[0], 

114 "error": str(error)[:120], 

115 "correct": False, 

116 } 

117 

118 

119def counts(items): 

120 """score/total/wrong/truncated/errors over `items`; truncation is not incorrectness.""" 

121 errors = sum(1 for i in items if "error" in i) 

122 return { 

123 "score": sum(1 for i in items if i.get("correct")), 

124 "total": len(items), 

125 "wrong": sum( 

126 1 

127 for i in items 

128 if not i.get("correct") and not i.get("truncated") and "error" not in i 

129 ), 

130 "truncated": sum(1 for i in items if i.get("truncated")), 

131 "errors": errors, 

132 } 

133 

134 

135def verdict(gate): 

136 """OK / INCONCLUSIVE / DEGRADED / BROKEN over integrity counts, else NO RESULT.""" 

137 if not gate or gate.get("total", 0) <= gate.get("errors", 0): 

138 return NO_RESULT 

139 if not gate.get("wrong"): 

140 return "INCONCLUSIVE" if gate.get("truncated") else "OK" 

141 return "BROKEN" if gate["wrong"] >= gate["total"] / 2 else "DEGRADED" 

142 

143 

144def summarise(items): 

145 """The report's correctness block, or None when every request errored.""" 

146 if all("error" in i for i in items): 

147 return None 

148 block = {**counts(items), "items": items} 

149 for kind in KINDS: 

150 group = [i for i in items if i.get("kind") == kind] 

151 if group: 

152 block[kind] = counts(group) 

153 block["verdict"] = verdict(block.get(INTEGRITY)) 

154 return block 

155 

156 

157def kind_of(item): 

158 """An item's recorded kind, else its prompt's kind today; unknown prompts are integrity.""" 

159 return item.get("kind") or _KIND_BY_PREVIEW.get(item.get("prompt"), INTEGRITY) 

160 

161 

162def by_kind(block): 

163 """{kind: counts} for a correctness block of any age; None without one.""" 

164 if not block: 

165 return None 

166 if INTEGRITY in block: 

167 return {kind: block[kind] for kind in KINDS if kind in block} 

168 items = block.get("items") or [] 

169 if not items: 

170 return {INTEGRITY: {field: block.get(field, 0) for field in COUNT_FIELDS}} 

171 groups = {} 

172 for item in items: 

173 groups.setdefault(kind_of(item), []).append(item) 

174 return {kind: counts(group) for kind, group in groups.items()} 

175 

176 

177def integrity(block): 

178 """The counts the verdict is taken over, or None when there are none.""" 

179 return (by_kind(block) or {}).get(INTEGRITY) 

180 

181 

182def outcomes(block, kind): 

183 """{prompt preview: correct} for the MEASURED (not truncated or errored) items of `kind`.""" 

184 return { 

185 item.get("prompt"): bool(item.get("correct")) 

186 for item in (block or {}).get("items") or [] 

187 if kind_of(item) == kind and "error" not in item and not item.get("truncated") 

188 } 

189 

190 

191def annotate(block): 

192 """`block` plus the kinds and verdict an older report never recorded, for the viewer.""" 

193 kinds = by_kind(block) 

194 if kinds is None or INTEGRITY in block: 

195 return block 

196 items = [{**item, "kind": kind_of(item)} for item in block.get("items") or []] 

197 return { 

198 **block, 

199 **kinds, 

200 "items": items, 

201 "verdict": verdict(kinds.get(INTEGRITY)), 

202 } 

203 

204 

205def exit_code(block): 

206 """--correctness-only's exit status, over the integrity items only. 

207 

208 1: unreachable or wrong; 2: cut off (re-run with a bigger budget); 0 otherwise. 

209 """ 

210 return {"OK": 0, "INCONCLUSIVE": 2}.get(verdict(integrity(block)), 1) 

211 

212 

213def _notes(block, gate, skill): 

214 """The NOTE lines under the item list.""" 

215 lines = [] 

216 truncated = block.get("truncated", 0) 

217 if truncated: 

218 lines += [ 

219 f" NOTE: {truncated} probe(s) ran out of tokens before answering. That is a", 

220 " MEASUREMENT limit, not a model fault -- raise --correctness-max-tokens.", 

221 ] 

222 if gate.get("wrong"): 

223 lines += [ 

224 " NOTE: a wrong INTEGRITY answer means broken kernels or an over-aggressive", 

225 " quant, not a slow model. Check the GGUF tensor types before tuning speed.", 

226 ] 

227 elif skill and skill.get("wrong"): 

228 lines += [ 

229 " NOTE: capability misses (tokenisation, trick reasoning, trivia) are the", 

230 " model's: compare with its own earlier report, not with a full score.", 

231 ] 

232 return lines 

233 

234 

235def print_correctness(block): 

236 """Render the probe: a model can be fast and wrong, so show both.""" 

237 print() 

238 kinds = by_kind(block) or {} 

239 gate = kinds.get(INTEGRITY) 

240 if gate is None: 

241 print(" Correctness probe: NO RESULT -- endpoint unreachable") 

242 return 

243 cut = f", {gate['truncated']} truncated" if gate.get("truncated") else "" 

244 print( 

245 f" Correctness probe: integrity {gate['score']}/{gate['total']} correct" 

246 f"{cut} [{verdict(gate)}]" 

247 ) 

248 skill = kinds.get(CAPABILITY) 

249 if skill: 

250 print( 

251 f" capability: {skill['score']}/{skill['total']} -- not a kernel verdict" 

252 ) 

253 for item in block.get("items") or []: 

254 kind = kind_of(item) 

255 if "error" in item: 

256 print(f" ERR {kind:<10} {item['prompt'][:42]:<42} {item['error'][:40]}") 

257 continue 

258 mark = "ok " if item["correct"] else "XX " 

259 got = item.get("answer_preview", "")[:44] 

260 print(f" {mark} {kind:<10} expected={item['expected']:<9} got={got!r}") 

261 for line in _notes(block, gate, skill): 

262 print(line)