Coverage for orchestrant/benchmark/openai_api.py: 60%

1#!/usr/bin/env python3 

2"""Benchmark any OpenAI-compatible LLM endpoint, with CPU and RAM tracking. 

3 

4Usage: 

5 orchestrant-bench speed 

6 orchestrant-bench speed --model gemma4:26b --prompts 10 

7 orchestrant-bench speed --output benchmark_results.json 

8""" 

9 

10import argparse 

11import json 

12import os 

13import sys 

14import threading 

15import time 

16import urllib.error 

17import urllib.request 

18from datetime import UTC, datetime 

19 

20from orchestrant.benchmark import answers, correctness, speed_summary 

21from orchestrant.benchmark.answers import accounting, from_body, read_stream 

22from orchestrant.benchmark.client import ( 

23 entry_config, 

24 post_json, 

25 request_headers, 

26 run_start, 

27) 

28from orchestrant.benchmark.correctness import CORRECTNESS_PROBES, print_correctness 

29from orchestrant.benchmark.energy import energy_block, energy_lines, renet 

30from orchestrant.benchmark.hostload import RequestBracket, open_meters, summary_lines 

31from orchestrant.benchmark.provenance import collect_or_error 

32 

33 

34# Hashed into tool_sha256: every file that decides what a speed row says. 

35SPEED_TOOL_FILES = ( 

36 "openai_api.py", 

37 "answers.py", 

38 "correctness.py", 

39 "energy.py", 

40 "hostload.py", 

41) 

42 

43 

44# LLM_BASE_URL is the name to use; OLLAMA_BASE_URL still works for existing scripts. 

45LLM_BASE_URL = ( 

46 os.environ.get("LLM_BASE_URL") 

47 or os.environ.get("OLLAMA_BASE_URL") 

48 or "http://localhost:11434" 

49) 

50OLLAMA_BASE_URL = LLM_BASE_URL # backwards-compatible alias 

51GLANCES_URL = os.environ.get("GLANCES_URL", "http://localhost:61208") 

52 

53 

54def collect_gpu_info(): 

55 """Name the accelerator that served the endpoint, if one is readable.""" 

56 try: 

57 from orchestrant.monitoring.gpu import GPUProbe 

58 

59 with GPUProbe() as probe: 

60 if not probe.available: 

61 return {} 

62 snapshot = probe.read() 

63 gpu = { 

64 "vendor": probe.vendor, 

65 "name": probe.gpu_name, 

66 "backend": probe.backend, 

67 } 

68 if probe.memory_type: 

69 gpu["memory_type"] = probe.memory_type 

70 if probe.memory_bandwidth_mbps: 

71 gpu["memory_bandwidth_gbps"] = round( 

72 probe.memory_bandwidth_mbps / 1000, 1 

73 ) 

74 if snapshot is not None: 

75 gpu["memory_total_mb"] = round(snapshot.memory_total_bytes / (1024**2)) 

76 gpu["memory_used_mb"] = round(snapshot.memory_used_bytes / (1024**2)) 

77 return gpu 

78 except Exception: 

79 return {} 

80 

81 

82def collect_hardware_info(): 

83 """Collect system hardware info for reproducibility, naming gaps in `incomplete`.""" 

84 info = {} 

85 info["timestamp"] = datetime.now(UTC).isoformat() 

86 

87 # OS info 

88 try: 

89 import platform 

90 

91 info["os"] = platform.system() 

92 info["os_release"] = platform.release() 

93 info["os_version"] = platform.version() 

94 info["architecture"] = platform.machine() 

95 info["processor"] = platform.processor() 

96 except Exception: # nosec B110 -- best-effort probe 

97 pass 

98 

99 # CPU info from /proc/cpuinfo (Linux) 

100 try: 

101 with open("/proc/cpuinfo") as f: 

102 cpuinfo = f.read() 

103 cpus = [p for p in cpuinfo.split("\n\n") if p.strip()] 

104 info["cpu_count"] = len(cpus) 

105 for line in cpus[0].split("\n") if cpus else []: 

106 if "model name" in line.lower(): 

107 info["cpu_model"] = line.split(":")[1].strip() 

108 break 

109 # Total cores per CPU (core id max + 1) 

110 core_ids = set() 

111 for cpu in cpus: 

112 for line in cpu.split("\n"): 

113 if "core id" in line.lower(): 

114 core_ids.add(line.split(":")[1].strip()) 

115 if core_ids: 

116 info["cpu_physical_cores"] = len(core_ids) 

117 # Thread(s) per core 

118 for line in cpus[0].split("\n") if cpus else []: 

119 if "siblings" in line.lower(): 

120 info["cpu_threads_per_core"] = int(line.split(":")[1].strip()) 

121 break 

122 info["cpu_total_threads"] = info["cpu_count"] 

123 except Exception: # nosec B110 -- best-effort probe 

124 pass 

125 

126 # RAM 

127 try: 

128 with open("/proc/meminfo") as f: 

129 mem = f.read() 

130 for line in mem.split("\n"): 

131 if line.startswith("MemTotal:"): 

132 info["ram_total_kb"] = int(line.split()[1]) 

133 info["ram_total_gb"] = round(info["ram_total_kb"] / (1024**2), 1) 

134 break 

135 except Exception: # nosec B110 -- best-effort probe 

136 pass 

137 

138 # Container info via cgroup 

139 try: 

140 with open("/proc/1/cgroup") as f: 

141 cgroup = f.read() 

142 info["in_container"] = ( 

143 "docker" in cgroup or "containerd" in cgroup or len(cgroup.splitlines()) > 0 

144 ) 

145 except Exception: 

146 info["in_container"] = None 

147 

148 # OLLAMA_HOST env 

149 info["ollama_host"] = os.environ.get("OLLAMA_HOST", "default") 

150 

151 # GPU identity, whichever vendor is present. 

152 gpu = collect_gpu_info() 

153 if gpu: 

154 info["gpu"] = gpu 

155 

156 # Portable fallback for what /proc could not provide (all of it on Windows and macOS) 

157 try: 

158 import platform as _pf 

159 

160 info.setdefault("os", _pf.system()) 

161 info.setdefault("os_release", _pf.release()) 

162 info.setdefault("architecture", _pf.machine()) 

163 if not info.get("processor"): 

164 info["processor"] = _pf.processor() or None 

165 except Exception: # nosec B110 -- best-effort probe 

166 pass 

167 try: 

168 import psutil as _ps 

169 

170 if not info.get("cpu_physical_cores"): 

171 info["cpu_physical_cores"] = _ps.cpu_count(logical=False) 

172 if not info.get("cpu_total_threads"): 

173 info["cpu_total_threads"] = _ps.cpu_count(logical=True) 

174 if not info.get("ram_total_gb"): 

175 info["ram_total_gb"] = round(_ps.virtual_memory().total / (1024**3), 1) 

176 except Exception: # nosec B110 -- best-effort probe 

177 pass 

178 

179 # Name the gaps: a benchmark whose host is unknown cannot be compared later. 

180 expected = ("os", "architecture", "cpu_total_threads", "ram_total_gb") 

181 missing = [k for k in expected if not info.get(k)] 

182 if missing: 

183 info["incomplete"] = missing 

184 

185 return info 

186 

187 

188# Prompts of varying length 

189 

190SHORT_PROMPTS = [ 

191 "What is 2+2?", 

192 "Say hello.", 

193 "What is the capital of France?", 

194 "Explain recursion in one sentence.", 

195 "What is an API?", 

196] 

197 

198MEDIUM_PROMPTS = [ 

199 "Write a Python function that computes the Fibonacci sequence using dynamic programming. Include a brief explanation of the time complexity.", 

200 "Explain the difference between REST and GraphQL APIs. When would you choose one over the other? Provide concrete examples.", 

201 "Write a bash script that monitors CPU and memory usage of a specific process every 5 seconds and logs the results to a CSV file.", 

202] 

203 

204# Named backends; sibling tools import these. See docs/source/benchmark.rst § Named backends 

205 

206 

207def _default_backends_file(): 

208 """Where the registry lives: LLM_BACKENDS, else the hub checkout's, else a local copy.""" 

209 env = os.environ.get("LLM_BACKENDS") 

210 if env: 

211 return env 

212 here = os.path.dirname(os.path.abspath(__file__)) 

213 repo = os.path.dirname(os.path.dirname(here)) 

214 hub = os.path.join( 

215 repo, "third_party", "ANTfrastructure", "linux", "llm-stack", "backends.json" 

216 ) 

217 return hub if os.path.isfile(hub) else os.path.join(here, "backends.json") 

218 

219 

220BACKENDS_FILE = _default_backends_file() 

221 

222 

223def load_backends(path=None): 

224 """Read backends.json; missing or malformed gives an empty registry, never raises.""" 

225 path = path or BACKENDS_FILE 

226 try: 

227 with open(path) as f: 

228 data = json.load(f) 

229 return data.get("backends", {}), data.get("default") 

230 except FileNotFoundError: 

231 return {}, None 

232 except Exception as e: 

233 print( 

234 f" WARNING: could not read {path} ({e}); named backends unavailable", 

235 file=sys.stderr, 

236 ) 

237 return {}, None 

238 

239 

240def resolve_backend(name=None, base_url=None, path=None): 

241 """Return (base_url, default_model, source) following the order above.""" 

242 backends, default_name = load_backends(path) 

243 

244 if base_url: 

245 return base_url.rstrip("/"), None, "--base-url" 

246 

247 env = os.environ.get("LLM_BASE_URL") or os.environ.get("OLLAMA_BASE_URL") 

248 if env: 

249 # A named backend still supplies its default model, if it matches. 

250 entry = backends.get(name) if name else None 

251 return env.rstrip("/"), (entry or {}).get("model"), "environment" 

252 

253 if name: 

254 if name not in backends: 

255 known = ", ".join(sorted(backends)) or "(none configured)" 

256 raise SystemExit(f"unknown backend {name!r}. Known: {known}") 

257 entry = backends[name] 

258 return entry["base_url"].rstrip("/"), entry.get("model"), f"backend {name!r}" 

259 

260 if default_name and default_name in backends: 

261 entry = backends[default_name] 

262 return ( 

263 entry["base_url"].rstrip("/"), 

264 entry.get("model"), 

265 f"default backend {default_name!r}", 

266 ) 

267 

268 return "http://localhost:11434", None, "built-in default" 

269 

270 

271def resolve_backend_entry(name=None, base_url=None, path=None): 

272 """The whole backends.json entry; a bare --base-url gets {} so no API key rides along.""" 

273 backends, default_name = load_backends(path) 

274 if name: 

275 if name not in backends: 

276 known = ", ".join(sorted(backends)) or "(none configured)" 

277 raise SystemExit(f"unknown backend {name!r}. Known: {known}") 

278 return dict(backends[name]) 

279 if base_url: 

280 return {} 

281 if default_name and default_name in backends: 

282 return dict(backends[default_name]) 

283 return {} 

284 

285 

286def print_backends(path=None): 

287 backends, default_name = load_backends(path) 

288 if not backends: 

289 print(" No backends configured (backends.json missing or empty).") 

290 return 

291 print("\n Configured backends:") 

292 for name in sorted(backends): 

293 entry = backends[name] 

294 mark = " (default)" if name == default_name else "" 

295 print(f" {name}{mark}") 

296 print(f" url: {entry['base_url']}") 

297 if entry.get("model"): 

298 print(f" model: {entry['model']}") 

299 if entry.get("note"): 

300 print(f" note: {entry['note']}") 

301 print() 

302 

303 

304LONG_PROMPTS = [ 

305 """Write a detailed technical blog post about building a production-ready LLM inference server. Cover the following aspects: 

3061. Model serving frameworks and their trade-offs (Ollama, vLLM, TGI, Triton Inference Server) 

3072. Hardware considerations: CPU vs GPU inference, memory requirements, quantization 

3083. API design: OpenAI-compatible endpoints, streaming, batching 

3094. Monitoring and observability: metrics to track, logging, alerting 

3105. Deployment strategies: Docker Compose, Kubernetes, auto-scaling 

3116. Security considerations: rate limiting, authentication, input sanitization 

312 

313For each section, provide practical recommendations based on real-world production experience.""", 

314] 

315 

316 

317def get_glances_data(endpoint): 

318 """Fetch JSON data from the Glances REST API (v4, falling back to v3).""" 

319 import requests 

320 

321 for api_ver in ("4", "3"): 

322 try: 

323 r = requests.get(f"{GLANCES_URL}/api/{api_ver}/{endpoint}", timeout=5) 

324 r.raise_for_status() 

325 return r.json() 

326 except Exception: # nosec B112 -- tries the next API version 

327 continue 

328 return None 

329 

330 

331def top_cpu_processes(limit=3): 

332 """Busiest processes since the PREVIOUS call: the port owner may not be the worker.""" 

333 try: 

334 import psutil 

335 except Exception: 

336 return [] 

337 procs = [] 

338 for p in psutil.process_iter(["name"]): 

339 try: 

340 pct = p.cpu_percent(None) 

341 # pid 0 is Windows' "System Idle Process": 737 % of idle ranked first. 

342 if pct > 0 and p.pid != 0: 

343 procs.append( 

344 { 

345 "pid": p.pid, 

346 "name": p.info.get("name") or "?", 

347 "cpu_percent": round(pct, 1), 

348 } 

349 ) 

350 except Exception: # nosec B112 -- skips a process that ended mid-read 

351 continue 

352 procs.sort(key=lambda d: d["cpu_percent"], reverse=True) 

353 return procs[:limit] 

354 

355 

356def sample_resources_psutil(): 

357 """Sample CPU and RAM via psutil on the host.""" 

358 import psutil 

359 

360 cpu = psutil.cpu_percent(interval=0.1) 

361 mem = psutil.virtual_memory() 

362 return { 

363 "cpu_percent": cpu, 

364 "ram_percent": mem.percent, 

365 "ram_used_gb": mem.used / (1024**3), 

366 "ram_total_gb": mem.total / (1024**3), 

367 } 

368 

369 

370def sample_resources_glances(): 

371 """Sample CPU and RAM via the Glances REST API.""" 

372 cpu_data = get_glances_data("cpu") 

373 mem_data = get_glances_data("mem") 

374 if cpu_data is None or mem_data is None: 

375 return None 

376 return { 

377 "cpu_percent": cpu_data.get("total", 0.0), 

378 "ram_percent": mem_data.get("percent", 0.0), 

379 "ram_used_gb": mem_data.get("used", 0) / (1024**3), 

380 "ram_total_gb": mem_data.get("total", 0) / (1024**3), 

381 } 

382 

383 

384_gpu_probe = None # one probe per process: driver contexts are not cheap 

385# Two first callers on two threads would each open a probe and leak one. 

386_gpu_probe_lock = threading.Lock() 

387 

388 

389def gpu_probe(): 

390 """The process-wide GPU probe, or None; False memoizes "none available".""" 

391 global _gpu_probe 

392 if _gpu_probe is None: 

393 with _gpu_probe_lock: 

394 if _gpu_probe is None: 

395 try: 

396 from orchestrant.monitoring.gpu import GPUProbe 

397 

398 probe = GPUProbe() 

399 _gpu_probe = probe if probe.available else False 

400 except Exception: 

401 _gpu_probe = False 

402 return _gpu_probe or None 

403 

404 

405def sample_gpu_resources(): 

406 """Local GPU utilization, VRAM and power; {} when unavailable, and never raises.""" 

407 try: 

408 probe = gpu_probe() 

409 if probe is None: 

410 return {} 

411 snapshot = probe.read() 

412 if snapshot is None: 

413 return {} 

414 return { 

415 "gpu_utilization_percent": round(snapshot.utilization, 1), 

416 "gpu_memory_used_gb": round(snapshot.memory_used_bytes / (1024**3), 2), 

417 "gpu_power_watts": round(snapshot.power_watts, 1), 

418 } 

419 except Exception: 

420 return {} 

421 

422 

423def _get_json(url, entry=None, timeout=5): 

424 """GET one JSON document with the backend entry's auth and headers.""" 

425 req = urllib.request.Request(url, headers=request_headers(entry)) 

426 try: 

427 with urllib.request.urlopen(req, timeout=timeout) as r: # nosec B310 -- endpoint URL from config 

428 return json.load(r) 

429 except urllib.error.HTTPError as e: 

430 # The error holds the response open until GC, and every caller only needs it raised. 

431 e.close() 

432 raise 

433 

434 

435def list_models_via_api(base_url=None, entry=None): 

436 """Every model id the endpoint lists: /v1/models, else Ollama's /api/tags.""" 

437 base = base_url or LLM_BASE_URL 

438 try: 

439 models = _get_json(f"{base}/v1/models", entry).get("data", []) 

440 if models: 

441 return [m["id"] for m in models] 

442 except Exception: # nosec B110 -- best-effort probe 

443 pass 

444 # Ollama-native fallback: /api/tags lists what is actually pulled. 

445 try: 

446 tags = _get_json(f"{base}/api/tags", entry).get("models", []) 

447 if tags: 

448 return [t.get("name") or t.get("model", "unknown") for t in tags] 

449 except Exception: # nosec B110 -- best-effort probe 

450 pass 

451 return [] 

452 

453 

454def detect_model_via_api(base_url=None, entry=None): 

455 """The model an endpoint serves, when that can be known.""" 

456 models = list_models_via_api(base_url, entry) 

457 # A listing is not a loaded model, and hot-loading a guess can crash an NPU lane. 

458 if len(models) > 1: 

459 shown = ", ".join(models[:6]) 

460 if len(models) > 6: 

461 shown += f", ... ({len(models)} in all)" 

462 raise SystemExit( 

463 f"{base_url or LLM_BASE_URL} lists {len(models)} models and does not " 

464 f"say which one is loaded: {shown}. Pass --model, or use a --backend " 

465 f"that names one." 

466 ) 

467 return models[0] if models else "unknown" 

468 

469 

470def resolve_model(explicit, backend_model, entry, detect=None): 

471 """--model, else the backend's default, else ask the endpoint (never a probe:false one).""" 

472 if explicit or backend_model: 

473 return explicit or backend_model 

474 if not (entry or {}).get("probe", True): 

475 raise SystemExit( 

476 "this backend is marked probe:false, so /v1/models is not queried. " 

477 'Pass --model, or give the entry a "model" field in backends.json.' 

478 ) 

479 return (detect or detect_model_via_api)(entry=entry) 

480 

481 

482def benchmark_chat( 

483 model, 

484 prompts, 

485 *, 

486 max_tokens=256, 

487 temperature=0.0, 

488 stream=False, 

489 extra_params=None, 

490 warmup=True, 

491 base_url=None, 

492 entry=None, 

493 lane=None, 

494 meter=None, 

495 idle_seconds=5.0, 

496): 

497 """Yield timing and resource rows per prompt from the chat completions endpoint. 

498 

499 `lane` and `meter` work only when this process shares a host with the server. 

500 """ 

501 endpoint = f"{base_url or LLM_BASE_URL}/v1/chat/completions" 

502 

503 # Warmup: one short request to load the model into memory 

504 if warmup: 

505 try: 

506 wp_payload = { 

507 "model": model, 

508 "messages": [{"role": "user", "content": "Warmup"}], 

509 "max_tokens": 1, 

510 } 

511 if extra_params: 

512 wp_payload.update(extra_params) 

513 with post_json(endpoint, wp_payload, entry=entry, timeout=120) as r: 

514 r.raw.read() 

515 except SystemExit: 

516 raise # a missing API key is a setup error, not a warmup hiccup 

517 except Exception: # nosec B110 -- best-effort warmup 

518 pass 

519 time.sleep(1) 

520 

521 idle_w = meter.idle_power(idle_seconds) if meter and meter.available else None 

522 

523 for i, prompt in enumerate(prompts): 

524 payload = { 

525 "model": model, 

526 "messages": [{"role": "user", "content": prompt}], 

527 "max_tokens": max_tokens, 

528 "temperature": temperature, 

529 "stream": stream, 

530 } 

531 if extra_params: 

532 payload.update(extra_params) 

533 # Request usage in streaming mode (Ollama supports this) 

534 if stream: 

535 payload["stream_options"] = {"include_usage": True} 

536 

537 # Measured over the request rather than sampled around it. 

538 bracket = RequestBracket( 

539 sample_resources, top_cpu_processes, lane, meter, idle_w 

540 ).start() 

541 start = time.monotonic() 

542 

543 try: 

544 if stream: 

545 with post_json( 

546 endpoint, payload, entry=entry, stream=True, timeout=300 

547 ) as response: 

548 reply = read_stream(response.lines()) 

549 else: 

550 with post_json(endpoint, payload, entry=entry, timeout=300) as r: 

551 reply = from_body(r.json()) 

552 except Exception as e: 

553 yield { 

554 "prompt_index": i, 

555 "prompt_preview": prompt[:60], 

556 "error": str(e), 

557 "latency_s": time.monotonic() - start, 

558 } 

559 continue 

560 

561 elapsed = time.monotonic() - start 

562 bracket.stop() 

563 

564 usage, streamed_chunks = reply.usage, reply.chunks 

565 prompt_tokens = usage.get("prompt_tokens", 0) if usage else 0 

566 completion_tokens = usage.get("completion_tokens", 0) if usage else 0 

567 total_tokens = usage.get("total_tokens", 0) if usage else 0 

568 

569 # Some servers ignore include_usage; counted content deltas are flagged as estimated. 

570 tokens_estimated = False 

571 if not completion_tokens and streamed_chunks: 

572 completion_tokens = streamed_chunks 

573 total_tokens = total_tokens or (prompt_tokens + completion_tokens) 

574 tokens_estimated = True 

575 

576 tokens_per_sec = completion_tokens / elapsed if elapsed > 0 else 0.0 

577 

578 # Split prefill from decode: the first token of any kind, thinking included, ends it. 

579 first = reply.first_token_at 

580 ttft = (first - start) if first is not None else None 

581 ttfa = reply.first_answer_at - start if reply.first_answer_at else None 

582 prefill_tps = prompt_tokens / ttft if ttft and prompt_tokens else None 

583 

584 # A reasoning model can be fastest per token and slowest to a usable answer. 

585 acct = accounting(reply, completion_tokens, max_tokens) 

586 

587 yield { 

588 "prompt_index": i, 

589 "prompt_preview": prompt[:60], 

590 "prompt_tokens": prompt_tokens, 

591 "completion_tokens": completion_tokens, 

592 "total_tokens": total_tokens, 

593 "tokens_estimated": tokens_estimated, 

594 "tokens_per_sec": round(tokens_per_sec, 2), 

595 # The metric to rank by; None when the budget ran out before an answer. 

596 "wall_s_to_answer": round(elapsed, 2) if acct["answered"] else None, 

597 "latency_s": round(elapsed, 2), 

598 "ttft_s": round(ttft, 3) if ttft is not None else None, 

599 "ttfa_s": round(ttfa, 3) if ttfa is not None else None, 

600 **answers.decode_fields(elapsed, ttft, completion_tokens), 

601 "prefill_tok_per_sec": round(prefill_tps, 1) if prefill_tps else None, 

602 **acct, 

603 "content_preview": (reply.content or reply.reasoning)[:80], 

604 **bracket.fields(completion_tokens), 

605 } 

606 

607 # A second idle baseline: idle draw drifts, so main() nets rows against the mean. 

608 if idle_w is not None and meter is not None: 

609 meter.idle_power(idle_seconds) 

610 

611 

612def run_correctness_probe( 

613 model, *, max_tokens=4000, extra_params=None, base_url=None, entry=None 

614): 

615 """Check the model still answers correctly at temperature 0; None if unreachable. 

616 

617 `max_tokens` is high because a reply cut off inside <think> scores WRONG. 

618 """ 

619 endpoint = f"{base_url or LLM_BASE_URL}/v1/chat/completions" 

620 items = [] 

621 for probe in CORRECTNESS_PROBES: 

622 payload = { 

623 "model": model, 

624 "messages": [{"role": "user", "content": probe.prompt}], 

625 "max_tokens": max_tokens, 

626 "temperature": 0, 

627 } 

628 if extra_params: 

629 payload.update(extra_params) 

630 try: 

631 with post_json(endpoint, payload, entry=entry, timeout=600) as r: 

632 content = r.json()["choices"][0]["message"]["content"] 

633 except Exception as e: 

634 items.append(correctness.errored_item(probe, e)) 

635 continue 

636 items.append(correctness.graded_item(probe, content)) 

637 return correctness.summarise(items) 

638 

639 

640_sampler_warned = False 

641# A Glances that did not answer is not asked again: each refused connect costs seconds. 

642_glances_down = False 

643 

644 

645def sample_resources(): 

646 """Unified resource sampler: Glances API → psutil (+ GPU) → zeros; never raises.""" 

647 global _sampler_warned, _glances_down 

648 gpu = sample_gpu_resources() 

649 result = None 

650 try: 

651 result = None if _glances_down else sample_resources_glances() 

652 except Exception: 

653 result = None # requests missing or Glances flaky: fall through to psutil 

654 _glances_down = result is None 

655 if result is None: 

656 try: 

657 result = sample_resources_psutil() 

658 except Exception as e: 

659 if not _sampler_warned: 

660 _sampler_warned = True 

661 print( 

662 f" WARNING: resource sampling failed ({e!r}); " 

663 "reporting zeros for CPU/RAM from now on", 

664 file=sys.stderr, 

665 ) 

666 result = { 

667 "cpu_percent": 0.0, 

668 "ram_percent": 0.0, 

669 "ram_used_gb": 0.0, 

670 "ram_total_gb": 0.0, 

671 } 

672 result.update(gpu) 

673 return result 

674 

675 

676def print_separator(char="━", width=80): 

677 print(char * width) 

678 

679 

680def print_table(results): 

681 """Print benchmark results as a formatted table.""" 

682 if not results: 

683 print("No results to display.") 

684 return 

685 

686 has_gpu = any( 

687 r.get("gpu_utilization_percent") is not None 

688 for r in results 

689 if "error" not in r 

690 ) 

691 headers = ["#", "Prompt", "PT", "CT", "T/s", "TTFT", "Ans(s)", "CPU%", "RAM(GB)"] 

692 col_widths = [3, 38, 5, 5, 7, 7, 8, 7, 8] 

693 if has_gpu: 

694 headers.append("GPU%") 

695 col_widths.append(6) 

696 

697 def fmt_row(values): 

698 return " ".join(f"{str(v)[:w]:{w}}{'':>1}" for v, w in zip(values, col_widths)) 

699 

700 print_separator("─") 

701 print(fmt_row(headers)) 

702 print_separator("─") 

703 

704 for r in results: 

705 if "error" in r: 

706 print( 

707 f" {r['prompt_index']:<3} {r['prompt_preview'][:38]:<38} ERROR: {r['error'][:40]}" 

708 ) 

709 continue 

710 ttft = r.get("ttft_s") 

711 values = [ 

712 r["prompt_index"], 

713 r["prompt_preview"][:38], 

714 r.get("prompt_tokens", "-"), 

715 r.get("completion_tokens", "-"), 

716 r.get("tokens_per_sec", "-"), 

717 f"{ttft:.2f}" if ttft is not None else "-", 

718 answers.answer_cell(r), 

719 r.get("cpu_percent", "-"), 

720 r.get("ram_used_gb", "-"), 

721 ] 

722 if has_gpu: 

723 values.append(r.get("gpu_utilization_percent", "-")) 

724 print(fmt_row(values)) 

725 

726 print_separator("─") 

727 print() 

728 

729 latencies = [ 

730 r["latency_s"] for r in results if "latency_s" in r and "error" not in r 

731 ] 

732 cpu_vals = [ 

733 r["cpu_percent"] for r in results if "cpu_percent" in r and "error" not in r 

734 ] 

735 ram_vals = [ 

736 r["ram_used_gb"] for r in results if "ram_used_gb" in r and "error" not in r 

737 ] 

738 speed = speed_summary.summarise(results) 

739 

740 if latencies: 

741 errored = f", {speed['errored']} errored" if speed["errored"] else "" 

742 print(f" Summary ({speed['requests']} requests{errored}):") 

743 print( 

744 f" Latency: {min(latencies):.2f}s / {sum(latencies) / len(latencies):.2f}s avg / {max(latencies):.2f}s max" 

745 ) 

746 print( 

747 f" CPU: {min(cpu_vals):.1f}% / {sum(cpu_vals) / len(cpu_vals):.1f}% avg / {max(cpu_vals):.1f}% max" 

748 ) 

749 print( 

750 f" RAM used: {min(ram_vals):.2f}GB / {sum(ram_vals) / len(ram_vals):.2f}GB avg / {max(ram_vals):.2f}GB max" 

751 ) 

752 gpu_vals = [ 

753 r["gpu_utilization_percent"] 

754 for r in results 

755 if r.get("gpu_utilization_percent") is not None 

756 ] 

757 if gpu_vals: 

758 print( 

759 f" GPU util: {min(gpu_vals):.1f}% / {sum(gpu_vals) / len(gpu_vals):.1f}% avg / {max(gpu_vals):.1f}% max" 

760 ) 

761 gpu_power_vals = [ 

762 r["gpu_power_watts"] 

763 for r in results 

764 if r.get("gpu_power_watts") is not None 

765 ] 

766 if gpu_power_vals: 

767 print( 

768 f" GPU power: {min(gpu_power_vals):.1f}W / {sum(gpu_power_vals) / len(gpu_power_vals):.1f}W avg / {max(gpu_power_vals):.1f}W max" 

769 ) 

770 

771 # speed_summary's figures, the ones `report` and the viewer print too. 

772 for line in speed_summary.summary_lines(speed): 

773 print(line)