Coverage for orchestrant/benchmark/report.py: 74%

1"""Summaries, tables and the viewer manifest over speed-run result files. 

2 

3Usage: 

4 python3 bench_report.py summary <result.json> 

5 python3 bench_report.py manifest <dir> <out.json> --title T --model M --generated TS 

6 python3 bench_report.py table <dir> 

7""" 

8 

9import argparse 

10import glob 

11import json 

12import os 

13import sys 

14 

15from orchestrant.benchmark import correctness, speed_summary 

16from orchestrant.benchmark.answers import answered_count, row_answer_s 

17 

18 

19def _rows(doc): 

20 """Every per-prompt result, errors included, whichever envelope the file uses.""" 

21 if "results" in doc: 

22 return list(doc["results"]) 

23 return [row for rep in doc.get("reports", []) for row in rep.get("results", [])] 

24 

25 

26def _mean(values): 

27 return sum(values) / len(values) if values else None 

28 

29 

30def summarise(doc): 

31 """One line's worth of numbers for a single result file, taken from speed_summary.""" 

32 rows = _rows(doc) 

33 ok = speed_summary.completed(rows) 

34 if not ok: 

35 return None 

36 return { 

37 **speed_summary.summarise(rows), 

38 # Cut rows have no time to an answer, so the mean travels with its row count. 

39 "answer_s": _mean([s for s in map(row_answer_s, ok) if s is not None]), 

40 "answered": answered_count(ok), 

41 "cpu_percent": _mean([r["cpu_percent"] for r in ok if "cpu_percent" in r]), 

42 "ram_used_gb": _mean([r["ram_used_gb"] for r in ok if "ram_used_gb" in r]), 

43 "gpu_utilization_percent": _mean( 

44 [ 

45 r["gpu_utilization_percent"] 

46 for r in ok 

47 if r.get("gpu_utilization_percent") is not None 

48 ] 

49 ), 

50 } 

51 

52 

53def result_files(directory): 

54 """Result files in `directory`, skipping our `_`-prefixed ones, which have no `results`.""" 

55 return sorted( 

56 f 

57 for f in glob.glob(os.path.join(directory, "*.json")) 

58 if not os.path.basename(f).startswith("_") 

59 ) 

60 

61 

62def build_manifest(directory, title, model, generated): 

63 """Index every result file for the viewer, with a `kind` so each renders as itself.""" 

64 manifest = { 

65 "title": title, 

66 "generated": generated, 

67 "model": model, 

68 "host_hardware": {}, 

69 "configs": [], 

70 } 

71 records = [] 

72 for path in result_files(directory): 

73 with open(path) as f: 

74 doc = json.load(f) 

75 records.append((doc.get("hardware"), doc.get("provenance"))) 

76 entry = { 

77 "label": os.path.basename(path)[:-5], 

78 "file": os.path.basename(path), 

79 "kind": report_kind(doc), 

80 "config": doc.get("config", {}), 

81 # The viewer cannot import the probe's table, so older reports get kinds here. 

82 "correctness": correctness.annotate(doc.get("correctness")), 

83 "results": doc.get("results", []), 

84 **run_fields(doc), 

85 } 

86 if entry["kind"] == "unknown": 

87 print( 

88 f" WARNING: {path} has neither 'benchmark' nor 'results' — " 

89 f"not a report this suite wrote", 

90 file=sys.stderr, 

91 ) 

92 if "reports" in doc: 

93 # A row without integer passed/total is not a score ("/ = 0%"). 

94 scored = [ 

95 { 

96 "label": r.get("label"), 

97 "model": r.get("model"), 

98 "passed": r.get("passed"), 

99 "total": r.get("total"), 

100 "effective_n": r.get("effective_n"), 

101 # Observed passing cases: passed * n / total is wrong for uneven attempts. 

102 "effective_k": r.get("effective_k"), 

103 "deterministic": r.get("deterministic"), 

104 "truncated": r.get("truncated"), 

105 "errored": r.get("errored"), 

106 "total_wall_s": r.get("total_wall_s"), 

107 "median_wall_s": r.get("median_wall_s"), 

108 "results": r.get("results", []), 

109 } 

110 for r in doc["reports"] 

111 if is_scored(r) 

112 ] 

113 unscored = [ 

114 {k: v for k, v in r.items() if k != "results"} 

115 for r in doc["reports"] 

116 if not is_scored(r) 

117 ] 

118 if scored: 

119 entry["scored"] = scored 

120 if unscored: 

121 entry["unscored"] = unscored 

122 entry["results"] = [ 

123 row for r in doc["reports"] for row in r.get("results", []) 

124 ] 

125 manifest["configs"].append(entry) 

126 manifest["host_hardware"] = host_hardware(records) 

127 return manifest 

128 

129 

130def host_hardware(records): 

131 """The Hardware card's record: the first real `hardware` block, else a provenance one.""" 

132 for which in (0, 1): 

133 for record in records: 

134 if isinstance(record[which], dict) and record[which]: 

135 return record[which] 

136 return {} 

137 

138 

139def run_fields(doc): 

140 """Per-run viewer fields beyond the rows: runtime, energy, threads, timestamp, speed.""" 

141 provenance = doc.get("provenance") 

142 if not isinstance(provenance, dict): 

143 provenance = {} 

144 hardware = doc.get("hardware") 

145 if not isinstance(hardware, dict): 

146 hardware = {} 

147 return { 

148 "backend": doc.get("backend"), 

149 "model": doc.get("model"), 

150 # The runtime is this endpoint's only: a lanes report records its first lane's. 

151 "base_url": provenance.get("base_url") or doc.get("api_url"), 

152 "runtime": provenance.get("runtime"), 

153 "energy": doc.get("energy"), 

154 "cpu_threads": hardware.get("cpu_total_threads"), 

155 "timestamp": provenance.get("timestamp_utc") or doc.get("timestamp"), 

156 "speed": speed_summary.summarise(_rows(doc)), 

157 } 

158 

159 

160def _count(value): 

161 return isinstance(value, int) and not isinstance(value, bool) 

162 

163 

164def is_scored(row): 

165 """A report row is a score only when passed AND total are integer counts.""" 

166 return _count(row.get("passed")) and _count(row.get("total")) 

167 

168 

169def report_kind(doc): 

170 """'benchmark' (shared envelope), 'throughput' (legacy shape) or 'unknown'.""" 

171 if doc.get("benchmark"): 

172 return doc["benchmark"] 

173 return "throughput" if "results" in doc else "unknown" 

174 

175 

176def comparison_rows(directory): 

177 rows = [] 

178 for path in result_files(directory): 

179 with open(path) as f: 

180 doc = json.load(f) 

181 s = summarise(doc) 

182 if s: 

183 rows.append((os.path.basename(path)[:-5], s)) 

184 return rows 

185 

186 

187def _fmt(value, spec, missing=" -"): 

188 return format(value, spec) if isinstance(value, (int, float)) else missing 

189 

190 

191def _answer(s, spec): 

192 """Mean seconds to an answer, with "(k/n)" whenever a reply was cut.""" 

193 k, n = s.get("answered") or (None, 0) 

194 cut = f" ({k}/{n} answered)" if n and k < n else "" 

195 return f"{_fmt(s['answer_s'], spec)}s{cut}" 

196 

197 

198def summary_line(s): 

199 """`report summary`: one run's figures under the speed runner's own names.""" 

200 return ( 

201 f" -> Decode: {_fmt(s['decode_tok_s'], '.1f')} tok/s " 

202 f"Overall: {_fmt(s['overall_tok_s'], '.1f')} tok/s " 

203 f"Answer: {_answer(s, '.1f')} avg " 

204 f"CPU: {_fmt(s['cpu_percent'], '.1f')}% " 

205 f"RAM: {_fmt(s['ram_used_gb'], '.1f')}GB" 

206 + (f" TTFT: {s['ttft_s']:.2f}s avg" if s["ttft_s"] is not None else "") 

207 + ( 

208 f" GPU: {s['gpu_utilization_percent']:.1f}%" 

209 if s["gpu_utilization_percent"] is not None 

210 else "" 

211 ) 

212 + (f" ERRORS: {s['errored']}" if s["errored"] else "") 

213 ) 

214 

215 

216def table_line(name, s): 

217 """`report table`: one result file per line, the figures of summary_line.""" 

218 return ( 

219 f" {name:25s} Decode: {_fmt(s['decode_tok_s'], '5.1f')} " 

220 f"Overall: {_fmt(s['overall_tok_s'], '5.1f')} " 

221 f"TTFT: {_fmt(s['ttft_s'], '5.2f')}s " 

222 f"Answer: {_answer(s, '5.1f')} " 

223 f"CPU: {_fmt(s['cpu_percent'], '5.1f')}% " 

224 f"CT: {s['completion_tokens']:4d} PT: {s['prompt_tokens']:4d}" 

225 + (f" ERRORS: {s['errored']}" if s["errored"] else "") 

226 ) 

227 

228 

229def main(): 

230 ap = argparse.ArgumentParser( 

231 description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter 

232 ) 

233 sub = ap.add_subparsers(dest="cmd", required=True) 

234 

235 p = sub.add_parser("summary") 

236 p.add_argument("file") 

237 p = sub.add_parser("manifest") 

238 p.add_argument("directory") 

239 p.add_argument("out") 

240 p.add_argument("--title", default="LLM Benchmark") 

241 p.add_argument("--model", default="") 

242 p.add_argument("--generated", default="") 

243 p = sub.add_parser("table") 

244 p.add_argument("directory") 

245 args = ap.parse_args() 

246 

247 if args.cmd == "summary": 

248 with open(args.file) as f: 

249 s = summarise(json.load(f)) 

250 if not s: 

251 print(" -> no successful results") 

252 return 

253 print(summary_line(s)) 

254 return 

255 

256 if args.cmd == "manifest": 

257 m = build_manifest(args.directory, args.title, args.model, args.generated) 

258 with open(args.out, "w") as f: 

259 json.dump(m, f, indent=2) 

260 print(f" Manifest: {args.out} ({len(m['configs'])} configs)") 

261 return 

262 

263 for name, s in comparison_rows(args.directory): 

264 print(table_line(name, s)) 

265 

266 

267if __name__ == "__main__": 

268 main()