Coverage for orchestrant/benchmark/speed_summary.py: 100%

1"""One speed summary per run: the headline figures every printer shows. 

2 

3On 2026-09-24 one report, the GenieX upgrade run's `v070-npu-speed.json`, read 

4three ways. The speed runner's table printed `Tokens/sec 18.3 avg`, `Overall 

5... 25.4 tok/s` and `Decode only 19.7 tok/s avg`; `report table` printed 

6`T/s: 18.3`; the viewer charted that 18.3 as "Tokens per Second (overall)", 

7the name the runner gave 25.4. Each printer averaged its own way: "Overall" 

8divided prompt AND completion tokens by the wall time (1763 tokens, 427 of 

9them the prompt), and every other figure was a mean of per-request rates, so 

10an 8-token reply weighed as much as a 256-token one. 

11 

12Every figure is now computed here, once. `print_table`, `report summary`, 

13`report table` and the viewer -- through the `speed` block `report manifest` 

14writes -- all read the dict `summarise` returns. Rates are ratios of sums, the 

15tokens over the seconds they took pooled across requests, as the CPU-rail 

16joules per token already were (hostload.summary_lines) and for the same 

17reason: a short request's timing noise must not outweigh a long request's 

18measurement. Times are means. 

19 

20Which rows count: every request that completed, cut at `max_tokens` or not, 

21thinking or not -- its tokens were produced and timed either way. Only an 

22errored request leaves the figures, and it is counted apart. Time to an 

23ANSWER is another matter, and answers.py owns it: a cut reply has none. 

24""" 

25 

26from __future__ import annotations 

27 

28 

29def completed(rows): 

30 """The requests that returned a reply: every row without an `error` key. 

31 

32 The key, not its value: the runner writes `str(e)`, which is "" for an 

33 exception raised without a message, and such a row still has no tokens or 

34 timings to count. 

35 """ 

36 return [r for r in rows if "error" not in r] 

37 

38 

39def _number(value): 

40 return isinstance(value, (int, float)) and not isinstance(value, bool) 

41 

42 

43def _seconds(row, n, rate_key, seconds_key): 

44 """The seconds behind one row's rate: recorded, or read back from the rate.""" 

45 if seconds_key and _number(row.get(seconds_key)): 

46 return row[seconds_key] 

47 rate = row.get(rate_key) 

48 return n / rate if _number(rate) and rate > 0 else None 

49 

50 

51def _pooled_rate(rows, amount, rate_key, seconds_key=None): 

52 """Sum of amounts over the sum of the seconds they took, or None. 

53 

54 Each row's seconds are its `seconds_key` field where it has one, else read 

55 back from its own rate (amount / rate): the runner computed the rate from 

56 unrounded times, while `latency_s` and `ttft_s` are stored rounded to 10 ms 

57 and 1 ms -- nearly 2 % of the 0.3 s decode window of an 8-token reply. 

58 """ 

59 pairs = [(amount(r), r) for r in rows] 

60 pairs = [(n, _seconds(r, n, rate_key, seconds_key)) for n, r in pairs if n > 0] 

61 pairs = [(n, s) for n, s in pairs if s is not None and s > 0] 

62 if not pairs: 

63 return None 

64 return sum(n for n, _ in pairs) / sum(s for _, s in pairs) 

65 

66 

67def decode_tok_s(rows): 

68 """Tokens decoded after the first, over the seconds spent decoding them. 

69 

70 Answers "how fast does this lane generate once it has started?" for the 

71 run as a whole: the sum of `completion_tokens - 1` over the sum of the 

72 decode windows (`decode_s`, or read back from `decode_tok_per_sec` in a 

73 report older than it) of the streamed rows of more than one token. A row 

74 whose window was too short for a rate of its own still counts, as it did 

75 when it carried one (answers.decode_fields). The first token belongs to 

76 the prefill, as in the row fields. 

77 

78 Pooled, not a mean of per-request rates: the GenieX v0.6.1 -> v0.7.0 NPU 

79 loss read -13.4 % as that mean, dragged by the 8- and 12-token replies 

80 whose rates moved -6.5 % and -8.6 %; pooled it reads -14.8 %, and the 

81 per-prompt median `bench_compare` prints is -14.7 %. On the 2048-token 

82 `v070r2-cpu-speed-answer` the mean said 19.2 tok/s for a run that decoded 

83 9928 tokens at 13.9: five replies of 144-815 tokens at 20-27 tok/s 

84 counted as much as the four of 1666-2048 at 11.8-13.4. 

85 """ 

86 return _pooled_rate( 

87 rows, 

88 lambda r: (r.get("completion_tokens") or 0) - 1, 

89 "decode_tok_per_sec", 

90 "decode_s", 

91 ) 

92 

93 

94def prefill_tok_s(rows): 

95 """Prompt tokens over the time to the first token, pooled across requests. 

96 

97 The sum of `prompt_tokens` over the sum of TTFTs, over the rows with a 

98 `prefill_tok_per_sec`. A TTFT carries the request's fixed overhead too, so 

99 on the speed runner's 11-158-token prompts this reads far below what the 

100 lane prefills at agent sizes (about 300 tok/s on the NPU lane, where the 

101 contract's cold 1.8k-token prefill measured about 1000): compare it between 

102 runs of one prompt set, not with a prefill benchmark. 

103 """ 

104 return _pooled_rate( 

105 rows, lambda r: r.get("prompt_tokens") or 0, "prefill_tok_per_sec" 

106 ) 

107 

108 

109def overall_tok_s(rows): 

110 """Completion tokens over the whole wall time: what a caller received. 

111 

112 Answers "sending these prompts one after another, how many tokens per 

113 second came back?": the sum of `completion_tokens` over the sum of 

114 `latency_s`, prefill, thinking and decode all inside the time. Prompt 

115 tokens are not output. The runner's old "Overall" line counted them and 

116 read 25.4 tok/s for a lane that generated 19.3, and a CPU lane's 31-32 

117 from that line was once published as its decode rate (the page's 

118 correction: ~30). A request that returned no tokens still cost its wall 

119 time, so it stays in the denominator. 

120 """ 

121 timed = [ 

122 r 

123 for r in rows 

124 if _number(r.get("completion_tokens")) and _number(r.get("latency_s")) 

125 ] 

126 seconds = sum(r["latency_s"] for r in timed) 

127 if seconds <= 0: 

128 return None 

129 return sum(r["completion_tokens"] for r in timed) / seconds 

130 

131 

132def ttft_s(rows): 

133 """Mean seconds to the first token of any kind, thinking included. 

134 

135 A time, not a rate, so a plain mean: what one request waits on average 

136 before anything arrives. Over the rows that measured one; None, not 0, 

137 when none did -- a non-streamed run has no first-token moment, and 0.00 s 

138 would claim an instant one. 

139 """ 

140 times = [r["ttft_s"] for r in rows if _number(r.get("ttft_s"))] 

141 return sum(times) / len(times) if times else None 

142 

143 

144def _spread(rows, key): 

145 """[lowest, highest] of one row field, or None: the per-request range.""" 

146 values = [r[key] for r in rows if _number(r.get(key))] 

147 return [min(values), max(values)] if values else None 

148 

149 

150def summarise(rows): 

151 """The run's headline figures, for every printer to read. 

152 

153 `rows` is the report's `results` as the runner wrote them, errors 

154 included. `per_request` keeps the range of each row field behind a 

155 headline, so a printer can show the spread without averaging it again. 

156 """ 

157 ok = completed(rows) 

158 return { 

159 "requests": len(ok), 

160 "errored": len(rows) - len(ok), 

161 # Counted from stream chunks where the server sent no usage. 

162 "estimated": sum(1 for r in ok if r.get("tokens_estimated")), 

163 "completion_tokens": sum(r.get("completion_tokens") or 0 for r in ok), 

164 "prompt_tokens": sum(r.get("prompt_tokens") or 0 for r in ok), 

165 "wall_s": sum(r["latency_s"] for r in ok if _number(r.get("latency_s"))), 

166 "overall_tok_s": overall_tok_s(ok), 

167 "decode_tok_s": decode_tok_s(ok), 

168 "prefill_tok_s": prefill_tok_s(ok), 

169 "ttft_s": ttft_s(ok), 

170 "per_request": { 

171 key: _spread(ok, key) 

172 for key in ("tokens_per_sec", "decode_tok_per_sec", "ttft_s") 

173 }, 

174 } 

175 

176 

177def _range(spread, spec): 

178 return f" (per request {spread[0]:{spec}}-{spread[1]:{spec}})" if spread else "" 

179 

180 

181def summary_lines(s): 

182 """The speed runner's token, rate and TTFT lines, from one summary.""" 

183 lines = [] 

184 per = s["per_request"] 

185 if s["overall_tok_s"] is not None: 

186 ct, n = s["completion_tokens"], s["requests"] 

187 lines.append(f" Completion tok: {ct} total / {ct / n:.1f} avg per req") 

188 lines.append( 

189 f" Overall: {s['overall_tok_s']:.1f} tok/s = {ct} tokens / " 

190 f"{s['wall_s']:.1f}s, prefill and thinking included" 

191 + _range(per["tokens_per_sec"], ".1f") 

192 ) 

193 if s["ttft_s"] is None: 

194 lines.append(" TTFT: not measured — re-run with --stream") 

195 else: 

196 low, high = per["ttft_s"] 

197 lines.append( 

198 f" TTFT: {low:.2f}s / {s['ttft_s']:.2f}s avg / {high:.2f}s max" 

199 ) 

200 if s["decode_tok_s"] is not None: 

201 lines.append( 

202 f" Decode: {s['decode_tok_s']:.1f} tok/s after each first token" 

203 + _range(per["decode_tok_per_sec"], ".1f") 

204 ) 

205 if s["prefill_tok_s"] is not None: 

206 lines.append( 

207 f" Prefill: {s['prefill_tok_s']:.0f} tok/s prompt tokens / TTFT, " 

208 "request overhead included" 

209 ) 

210 if s["estimated"]: 

211 lines.append( 

212 f" Estimated: {s['estimated']} of {s['requests']} token counts " 

213 "are stream chunks: the server sent no usage" 

214 ) 

215 return lines