Coverage for orchestrant/benchmark/provenance.py: 87%

1#!/usr/bin/env python3 

2"""Record WHAT produced a measurement, so two reports can be compared later.""" 

3 

4import glob 

5import hashlib 

6import json 

7import os 

8import pathlib 

9import platform 

10import shutil 

11import subprocess # nosec B404 -- runs local tools, argv built here 

12import sys 

13import sysconfig 

14import time 

15import urllib.request 

16from concurrent.futures import ThreadPoolExecutor 

17from datetime import UTC, datetime 

18 

19from orchestrant.benchmark.client import redact_argv 

20 

21# Re-exported: tools fingerprint determinism.py without fingerprinting this plumbing. 

22from orchestrant.benchmark.determinism import ( # noqa: F401 

23 PROBE_PROMPT, 

24 SPACER_PROMPT, 

25 determinism_probe, 

26) 

27from orchestrant.benchmark.gateway import gateway_block, gateway_notes, lane_behind 

28 

29 

30SCHEMA_VERSION = 1 

31 

32 

33def _git(*args): 

34 try: 

35 out = subprocess.run( # nosec B603 B607 -- argv built here, tool from PATH, no shell 

36 ["git", *args], 

37 capture_output=True, 

38 text=True, 

39 timeout=10, 

40 cwd=os.path.dirname(os.path.abspath(__file__)), 

41 ) 

42 return out.stdout.strip() or None if out.returncode == 0 else None 

43 except Exception: 

44 return None 

45 

46 

47def _server_models(base_url, timeout=5): 

48 """Model ids the endpoint advertises, which says nothing about what is LOADED.""" 

49 try: 

50 with urllib.request.urlopen(f"{base_url}/v1/models", timeout=timeout) as r: # nosec B310 -- endpoint URL from config 

51 return sorted(m["id"] for m in json.load(r).get("data", [])) 

52 except Exception: 

53 return None 

54 

55 

56_GENIEX_VERSION_KEYS = { 

57 "geniex cli version": "cli", 

58 "qairt runtime version": "qairt", 

59 "llamacpp runtime hash": "llama_cpp", 

60} 

61 

62 

63def parse_geniex_version(text): 

64 """`geniex --version` -> {"cli", "qairt", "llama_cpp"}; absent lines stay absent.""" 

65 out = {} 

66 for line in (text or "").splitlines(): 

67 name, sep, value = line.partition(":") 

68 key = _GENIEX_VERSION_KEYS.get(name.strip().lower()) 

69 if sep and key and value.strip(): 

70 out[key] = value.strip() 

71 return out 

72 

73 

74def _installed_geniex(): 

75 """The GenieX CLI a lane here would run, or None; from WSL2 the Windows side first.""" 

76 candidates = [] 

77 local = os.environ.get("LOCALAPPDATA") 

78 if local: 

79 candidates.append(os.path.join(local, "GenieX CLI", "geniex.exe")) 

80 candidates += sorted( 

81 glob.glob("/mnt/c/Users/*/AppData/Local/GenieX CLI/geniex.exe") 

82 ) 

83 found = shutil.which("geniex") 

84 if found: 

85 candidates.append(found) 

86 return next((c for c in candidates if os.path.isfile(c)), None) 

87 

88 

89def _geniex_version(exe, timeout=30): 

90 try: 

91 out = subprocess.run( # nosec B603 -- argv built here, no shell 

92 [exe, "--version", "--skip-update"], 

93 capture_output=True, 

94 text=True, 

95 timeout=timeout, 

96 ) 

97 except Exception: 

98 return None 

99 return parse_geniex_version(out.stdout + out.stderr) or None 

100 

101 

102def _ollama_version(base_url, timeout=3): 

103 try: 

104 with urllib.request.urlopen(f"{base_url}/api/version", timeout=timeout) as r: # nosec B310 

105 return json.load(r).get("version") 

106 except Exception: 

107 return None 

108 

109 

110def _serves_geniex_root(base_url, timeout=3): 

111 """Does the root page look like GenieX's Swagger UI? Too weak to yield `verified`.""" 

112 try: 

113 with urllib.request.urlopen(f"{base_url}/", timeout=timeout) as r: # nosec B310 

114 return b"swagger-ui" in r.read(4096) 

115 except Exception: 

116 return False 

117 

118 

119# What served: the model files, the drivers, the host's own view 

120 

121# <cache>/<org>/<repo>/geniex.json maps each variant of a model id to its file. 

122_GENIEX_CACHE = (".cache", "geniex", "models") 

123 

124# Multi-GB GGUFs share their first MiB across quants: hash the head plus sampled windows. 

125_HEAD_BYTES = 1 << 20 

126_SAMPLES, _SAMPLE_BYTES = 16, 1 << 16 

127 

128 

129def _sampled_sha256(f, size): 

130 """sha256 over _SAMPLES windows at offsets i * (size - window) // (n - 1).""" 

131 h = hashlib.sha256() 

132 span = max(size - _SAMPLE_BYTES, 0) 

133 for i in range(_SAMPLES): 

134 f.seek(span * i // (_SAMPLES - 1)) 

135 h.update(f.read(_SAMPLE_BYTES)) 

136 return h.hexdigest() 

137 

138 

139def _file_identity(path, manifest_size=None, whole=False): 

140 """Name, size, mtime and a sha256 -- whole, or of the first MiB and a sample.""" 

141 ident = {"name": os.path.basename(path)} 

142 try: 

143 with open(path, "rb") as f: 

144 st = os.fstat(f.fileno()) 

145 digest = hashlib.sha256(f.read(None if whole else _HEAD_BYTES)).hexdigest() 

146 sampled = None if whole else _sampled_sha256(f, st.st_size) 

147 except OSError as e: 

148 return {**ident, "error": f"{type(e).__name__}: {e}"[:160]} 

149 ident["size"] = st.st_size 

150 ident["modified_utc"] = datetime.fromtimestamp(st.st_mtime, UTC).isoformat( 

151 timespec="seconds" 

152 ) 

153 if whole: 

154 ident["sha256"] = digest 

155 else: 

156 ident.update( 

157 head_sha256=digest, 

158 head_bytes=_HEAD_BYTES, 

159 sampled_sha256=sampled, 

160 sampled=f"{_SAMPLES} x {_SAMPLE_BYTES} bytes, evenly to the end", 

161 ) 

162 ident["size_matches_manifest"] = ( 

163 None if manifest_size is None else st.st_size == manifest_size 

164 ) 

165 return ident 

166 

167 

168def _read_json(path): 

169 """A JSON object from `path`, or None: a missing or broken file is a gap.""" 

170 try: 

171 with open(path, encoding="utf-8") as f: 

172 data = json.load(f) 

173 except (OSError, ValueError): 

174 return None 

175 return data if isinstance(data, dict) else None 

176 

177 

178def _geniex_cache_roots(local_only=False): 

179 """The model caches to look in: this OS's, and from WSL2 the Windows host's first.""" 

180 local = str(pathlib.Path.home().joinpath(*_GENIEX_CACHE)) 

181 if local_only or sys.platform == "win32": 

182 return [local] 

183 windows = sorted(glob.glob("/mnt/c/Users/*/" + "/".join(_GENIEX_CACHE))) 

184 return [*windows, local] 

185 

186 

187def _find_manifest(roots, org, repo): 

188 """(folder, geniex.json) for org/repo in the first root that has it, or None.""" 

189 for root in roots: 

190 folder = os.path.join(root, org, repo) 

191 manifest = _read_json(os.path.join(folder, "geniex.json")) 

192 if manifest is not None: 

193 return folder, manifest 

194 return None 

195 

196 

197def _manifest_sizes(manifest): 

198 """{file name: the size geniex.json recorded when it was downloaded}.""" 

199 entries = [ 

200 *(manifest.get("ModelFile") or {}).values(), 

201 manifest.get("MMProjFile") or {}, 

202 *(manifest.get("ExtraFiles") or []), 

203 ] 

204 return { 

205 e["Name"]: e.get("Size") 

206 for e in entries 

207 if isinstance(e, dict) and e.get("Name") 

208 } 

209 

210 

211def _pick_variant(variants, variant): 

212 """The ModelFile key a model id names: exactly, ignoring case, or the only one.""" 

213 if variant is None: 

214 return next(iter(variants)) if len(variants) == 1 else None 

215 if variant in variants: 

216 return variant 

217 return next((k for k in variants if k.lower() == variant.lower()), None) 

218 

219 

220def _qairt_bundle(folder, sizes): 

221 """A QAIRT bundle's genie_config.json (hashed whole: edited in place), ctx-bins and more.""" 

222 path = os.path.join(folder, "genie_config.json") 

223 dialog = (_read_json(path) or {}).get("dialog") or {} 

224 engine = dialog.get("engine") or {} 

225 bins = ((engine.get("model") or {}).get("binary") or {}).get("ctx-bins") or [] 

226 ext = (engine.get("backend") or {}).get("extensions") 

227 meta = _read_json(os.path.join(folder, "metadata.json")) or {} 

228 return { 

229 "files": [_file_identity(os.path.join(folder, b), sizes.get(b)) for b in bins], 

230 "genie_config": { 

231 **_file_identity(path, sizes.get("genie_config.json"), whole=True), 

232 "sampler": dialog.get("sampler"), 

233 "context_size": (dialog.get("context") or {}).get("size"), 

234 }, 

235 "backend_extensions": _file_identity( 

236 os.path.join(folder, ext), sizes.get(ext), whole=True 

237 ) 

238 if ext 

239 else None, 

240 "bundle_qairt": (meta.get("tool_versions") or {}).get("qairt"), 

241 "precision": meta.get("precision"), 

242 # No weights named is a gap to say, not an empty list to trust. 

243 "error": None if bins else "genie_config.json names no ctx-bins", 

244 } 

245 

246 

247def _model_files(model, roots): 

248 repo_id, _, variant = model.partition(":") 

249 org, _, repo = repo_id.partition("/") 

250 out = { 

251 "model": model, 

252 "cache_dir": None, 

253 "plugin": None, 

254 "variant": None, 

255 "files": [], 

256 "error": None, 

257 } 

258 found = _find_manifest(roots, org, repo) if org and repo else None 

259 if found is None: 

260 return {**out, "error": f"no geniex.json for {repo_id!r} in {roots}"} 

261 folder, manifest = found 

262 variants = manifest.get("ModelFile") or {} 

263 key = _pick_variant(variants, variant or None) 

264 out.update(cache_dir=folder, plugin=manifest.get("PluginId"), variant=key) 

265 if key is None: 

266 why = f"variant {variant!r} is not" if variant else "the id names no variant of" 

267 return {**out, "error": f"{why} the manifest's {sorted(variants)}"} 

268 sizes = _manifest_sizes(manifest) 

269 if out["plugin"] == "qairt": 

270 return {**out, **_qairt_bundle(folder, sizes)} 

271 names = [variants[key].get("Name"), (manifest.get("MMProjFile") or {}).get("Name")] 

272 out["files"] = [ 

273 _file_identity(os.path.join(folder, n), sizes.get(n)) for n in names if n 

274 ] 

275 return out 

276 

277 

278def geniex_model_files(model, roots=None): 

279 """WHICH files a GenieX model id resolves to in the lane's cache; never raises.""" 

280 if not model: 

281 return None 

282 try: 

283 return _model_files(model, roots or _geniex_cache_roots()) 

284 except Exception as e: # a manifest in a shape this does not know 

285 return {"model": model, "error": f"{type(e).__name__}: {e}"[:200]} 

286 

287 

288# Setup classes whose Control\Class keys give driver versions without admin or a subprocess. 

289_DRIVER_CLASSES = { 

290 "npu": "{f01a9d53-3ff6-48d2-9f97-c8a7004be10c}", # ComputeAccelerator: Hexagon 

291 "gpu": "{4d36e968-e325-11ce-bfc1-08002be10318}", # Display: Adreno 

292} 

293_DRIVER_FIELDS = { 

294 "DriverDesc": "name", 

295 "DriverVersion": "version", 

296 "DriverDate": "date", 

297 "ProviderName": "provider", 

298 "InfPath": "inf", 

299} 

300 

301 

302def _driver_row(winreg, parent, sub): 

303 """One installed driver's name, version, date, provider and INF, or None.""" 

304 row = {} 

305 try: 

306 with winreg.OpenKey(parent, sub) as key: 

307 for value, field in _DRIVER_FIELDS.items(): 

308 try: 

309 row[field] = str(winreg.QueryValueEx(key, value)[0]) 

310 except OSError: # a value this driver's INF does not set 

311 row[field] = None 

312 except OSError: # a device key this user may not read 

313 return None 

314 return row 

315 

316 

317def _class_drivers(winreg, guid): 

318 """Every non-Microsoft driver installed in one device setup class.""" 

319 base = rf"SYSTEM\CurrentControlSet\Control\Class\{guid}" 

320 rows = [] 

321 with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, base) as cls: 

322 for i in range(winreg.QueryInfoKey(cls)[0]): 

323 sub = winreg.EnumKey(cls, i) 

324 row = _driver_row(winreg, cls, sub) if sub.isdigit() else None 

325 # Microsoft's are the remote and basic display adapters, never a lane's. 

326 if row and row["version"] and row["provider"] != "Microsoft": 

327 rows.append(row) 

328 return rows 

329 

330 

331def driver_versions(): 

332 """The NPU and GPU driver versions under a Windows lane, or None with the reason.""" 

333 out = {"npu": None, "gpu": None, "source": None, "reason": None} 

334 if sys.platform != "win32": 

335 out["reason"] = ( 

336 f"not a Windows host ({sys.platform}); from WSL2 the host's drivers " 

337 f"are invisible, and a lane-runtime file written on the host " 

338 f"({LANE_RUNTIMES_ENV}) carries them" 

339 ) 

340 return out 

341 import winreg 

342 

343 out["source"] = r"HKLM\SYSTEM\CurrentControlSet\Control\Class" 

344 for kind, guid in _DRIVER_CLASSES.items(): 

345 try: 

346 out[kind] = _class_drivers(winreg, guid) 

347 except OSError as e: 

348 out["reason"] = f"{kind}: {type(e).__name__}: {e}"[:160] 

349 return out 

350 

351 

352def _geniex_serving(model, roots=None): 

353 """What a GenieX runtime adds: the served model's files and the drivers.""" 

354 return { 

355 "model_files": geniex_model_files(model, roots), 

356 "drivers": driver_versions(), 

357 } 

358 

359 

360# WSL2 cannot see the lane process; this names a lane-runtime file written on Windows. 

361LANE_RUNTIMES_ENV = "LLM_LANE_RUNTIMES" 

362 

363# Lanes restart every round, so an older snapshot keeps its data but loses `verified`. 

364SNAPSHOT_MAX_AGE_S = 12 * 3600 

365 

366 

367def lane_runtimes(lanes): 

368 """{name: runtime_info(url, model)} for {name: (url, model)}; never raises.""" 

369 out = {} 

370 for name, (url, model) in lanes.items(): 

371 try: 

372 out[name] = runtime_info(url, model) 

373 except Exception as e: 

374 out[name] = {"error": f"{type(e).__name__}: {e}"[:200]} 

375 return out 

376 

377 

378def write_lane_runtimes(path, lanes): 

379 """Snapshot what serves each lane, on the host that can see the lane processes. 

380 

381 Run it on Windows once the lanes are up, then export 

382 LLM_LANE_RUNTIMES=<the file's /mnt/c path> in WSL2. 

383 """ 

384 runtimes = lane_runtimes(lanes) 

385 doc = { 

386 "schema_version": SCHEMA_VERSION, 

387 "kind": "lane_runtimes", 

388 "captured_utc": datetime.now(UTC).isoformat(), 

389 "host": platform.node() or None, 

390 "lanes": [ 

391 {"lane": name, "base_url": url, "model": model, "runtime": runtimes[name]} 

392 for name, (url, model) in lanes.items() 

393 ], 

394 } 

395 tmp = f"{path}.tmp" 

396 with open(tmp, "w", encoding="utf-8") as f: 

397 json.dump(doc, f, indent=2) 

398 os.replace(tmp, path) 

399 return doc 

400 

401 

402def _names_a_server(entry): 

403 """A snapshot entry the host attributed: {"error": ...} or null is no evidence.""" 

404 runtime = entry.get("runtime") if isinstance(entry, dict) else None 

405 return isinstance(runtime, dict) and bool(runtime.get("server")) 

406 

407 

408def _snapshot_entry(entries, base_url): 

409 """The entry for `base_url`: exactly, else the only loopback one on its port.""" 

410 from orchestrant.benchmark.hostload import _port 

411 

412 def port(u): 

413 try: 

414 return _port(u) 

415 except ValueError: # a malformed port in a hand-edited file 

416 return None 

417 

418 url = (base_url or "").rstrip("/") 

419 entries = [e for e in entries if _names_a_server(e)] 

420 exact = [e for e in entries if (e.get("base_url") or "").rstrip("/") == url] 

421 by_port = [e for e in entries if port(url) and port(e.get("base_url")) == port(url)] 

422 return exact[0] if exact else (by_port[0] if len(by_port) == 1 else None) 

423 

424 

425def _age_s(stamp): 

426 try: 

427 return round( 

428 (datetime.now(UTC) - datetime.fromisoformat(stamp)).total_seconds() 

429 ) 

430 except (TypeError, ValueError): 

431 return None 

432 

433 

434def _installed_cli_mismatch(runtime): 

435 """The installed GenieX's version when it is not the snapshot lane's, else None.""" 

436 if runtime.get("server") != "geniex": 

437 return None 

438 installed = _installed_geniex() 

439 cli = (_geniex_version(installed) or {}).get("cli") if installed else None 

440 return cli if cli and cli != runtime.get("cli") else None 

441 

442 

443def load_lane_runtime(base_url, path=None, model=None, max_age_s=SNAPSHOT_MAX_AGE_S): 

444 """The lane-runtime file's runtime for `base_url` (`path` or $LLM_LANE_RUNTIMES), or None.""" 

445 path = path or os.environ.get(LANE_RUNTIMES_ENV) 

446 doc = (_read_json(path) if path else None) or {} 

447 entry = _snapshot_entry(doc.get("lanes") or [], base_url) or {} 

448 runtime = dict(entry.get("runtime") or {}) 

449 if not runtime: 

450 return None 

451 captured, age = doc.get("captured_utc"), _age_s(doc.get("captured_utc")) 

452 fresh = age is not None and age <= max_age_s 

453 mismatch = _installed_cli_mismatch(runtime) 

454 runtime["verified"] = bool(runtime.get("verified")) and fresh and not mismatch 

455 runtime["source"] = f"lane-runtime file {path}: {runtime.get('source')}" 

456 runtime["snapshot"] = { 

457 "path": path, 

458 "lane": entry.get("lane"), 

459 "host": doc.get("host"), 

460 "captured_utc": captured, 

461 "age_s": age, 

462 "stale": not fresh, 

463 "installed_cli_mismatch": mismatch, 

464 } 

465 return _with_model_files(runtime, model) 

466 

467 

468def _with_model_files(runtime, model): 

469 """`runtime`, its GenieX model files resolved again when another id served.""" 

470 served = (runtime.get("model_files") or {}).get("model") 

471 if model and runtime.get("server") == "geniex" and served != model: 

472 runtime["model_files"] = geniex_model_files(model) 

473 return runtime 

474 

475 

476def _file_keys(model_files): 

477 """({name: (size, head, sample)} of the files read, {every name}); a copy moves only mtime.""" 

478 files = [f for f in model_files.get("files") or [] if isinstance(f, dict)] 

479 read = { 

480 f.get("name"): (f.get("size"), f.get("head_sha256"), f.get("sampled_sha256")) 

481 for f in files 

482 if f.get("size") is not None 

483 } 

484 return read, {f.get("name") for f in files} 

485 

486 

487def _changed_files(old, new): 

488 """Names listed on one side only, or read on both with another identity.""" 

489 (was, was_names), (now, now_names) = _file_keys(old), _file_keys(new) 

490 moved = {n for n in was.keys() & now.keys() if was[n] != now[n]} 

491 return sorted(str(n) for n in (was_names ^ now_names) | moved) 

492 

493 

494def _sha_moved(old, new, key): 

495 """Both sides hashed the `key` file whole, and the hashes differ.""" 

496 was = (old.get(key) or {}).get("sha256") 

497 now = (new.get(key) or {}).get("sha256") 

498 return bool(was and now and was != now) 

499 

500 

501def _model_files_of(runtime): 

502 return (runtime or {}).get("model_files") or {} 

503 

504 

505def model_files_notes(old_rt, new_rt): 

506 """Other weights or another bundle config behind the same model id, for compare().""" 

507 old, new = _model_files_of(old_rt), _model_files_of(new_rt) 

508 if not old.get("files") or old.get("model") != new.get("model"): 

509 return [] 

510 notes = [] 

511 changed = _changed_files(old, new) 

512 if new.get("files") and changed: 

513 notes.append( 

514 f"MODEL FILES CHANGED behind {old['model']}: {', '.join(changed)} — " 

515 f"a re-pull or another file under the same model id" 

516 ) 

517 if _sha_moved(old, new, "genie_config"): 

518 notes.append( 

519 f"the QAIRT bundle's genie_config.json changed (sampler " 

520 f"{old['genie_config'].get('sampler')} → " 

521 f"{new['genie_config'].get('sampler')})" 

522 ) 

523 if _sha_moved(old, new, "backend_extensions"): 

524 notes.append( 

525 "the QAIRT bundle's HTP extensions file changed — it sets the " 

526 "NPU's perf profile, which moves speed under the same weights" 

527 ) 

528 return notes 

529 

530 

531def main(argv=None): 

532 """Write a lane-runtime file: `python -m orchestrant.benchmark.provenance`.""" 

533 import argparse 

534 

535 from orchestrant.benchmark.lanes import resolve_lane 

536 

537 ap = argparse.ArgumentParser( 

538 description=write_lane_runtimes.__doc__, 

539 formatter_class=argparse.RawDescriptionHelpFormatter, 

540 ) 

541 ap.add_argument( 

542 "lanes", 

543 nargs="+", 

544 type=resolve_lane, 

545 metavar="BACKEND|name=URL,model=MODEL", 

546 help="a backends.json name (geniex-npu) or a full name=URL,model=MODEL spec", 

547 ) 

548 ap.add_argument("--output", required=True, help="the JSON file to write") 

549 args = ap.parse_args(argv) 

550 doc = write_lane_runtimes(args.output, {n: (u, m) for n, u, m in args.lanes}) 

551 for e in doc["lanes"]: 

552 rt = e["runtime"] or {} 

553 print(f" {e['lane']:12s} {runtime_label(rt)} verified={rt.get('verified')}") 

554 print(f" Written to {args.output}; in WSL2 export {LANE_RUNTIMES_ENV}=<its path>") 

555 return 0 

556 

557 

558def _lane_process_runtime(proc, exe, model): 

559 """A GenieX lane process seen on this host: the exact binary and its flags.""" 

560 return { 

561 "server": "geniex", 

562 **(_geniex_version(exe) or {}), 

563 "serve_args": (proc.get("cmdline") or [])[1:], 

564 # Different start times are different launches, even with identical flags. 

565 "started": proc.get("started"), 

566 "verified": True, 

567 "source": f"lane process pid {proc['pid']}: {exe} --version", 

568 # A process on this OS reads this OS's model cache. 

569 **_geniex_serving(model, _geniex_cache_roots(local_only=True)), 

570 } 

571 

572 

573def runtime_info(base_url, model=None): 

574 """WHICH server build produced a measurement, and launched with WHICH flags. 

575 

576 Best evidence first: the port's listener (`verified`), a lane-runtime file, 

577 then the installed GenieX (`verified: False`). 

578 """ 

579 if not base_url: 

580 return None 

581 # A gateway alias answers for its lane, as a direct run of that lane would. 

582 base_url, model = lane_behind(base_url, model) or (base_url, model) 

583 from orchestrant.benchmark.hostload import LaneProcess, _port 

584 

585 lane = LaneProcess(base_url) 

586 proc = lane.info() if lane.available else None 

587 exe = (proc or {}).get("exe") or "" 

588 # Either separator: os.path on Linux does not split a Windows path at backslashes. 

589 name = os.path.basename(exe.replace("\\", "/")).lower() 

590 if proc is not None and name.startswith("geniex"): 

591 return _lane_process_runtime(proc, exe, model) 

592 snapshot = load_lane_runtime(base_url, model=model) 

593 if snapshot: 

594 return snapshot 

595 version = _ollama_version(base_url) 

596 if version: 

597 return { 

598 "server": "ollama", 

599 "version": version, 

600 "verified": True, 

601 "source": "/api/version", 

602 } 

603 # Only a lane that ANSWERS may be attributed to the installed binary. 

604 installed = None 

605 if ( 

606 _port(base_url) 

607 and _server_models(base_url, timeout=2) is not None 

608 and _serves_geniex_root(base_url) 

609 ): 

610 installed = _installed_geniex() 

611 if installed: 

612 return { 

613 "server": "geniex", 

614 **(_geniex_version(installed) or {}), 

615 "serve_args": None, 

616 "verified": False, 

617 "source": f"installed binary {installed}; {lane.reason}", 

618 **_geniex_serving(model), 

619 } 

620 return None 

621 

622 

623def runtime_label(runtime): 

624 """One line naming a runtime, for notes and tables.""" 

625 if not runtime: 

626 return "unknown" 

627 if runtime.get("server") == "geniex": 

628 return ( 

629 f"geniex {runtime.get('cli', '?')} (QAIRT {runtime.get('qairt', '?')}, " 

630 f"llama.cpp {runtime.get('llama_cpp', '?')})" 

631 ) 

632 return f"{runtime.get('server', '?')} {runtime.get('version', '?')}" 

633 

634 

635def _runtime_key(runtime): 

636 keys = ("server", "cli", "qairt", "llama_cpp", "version") 

637 return tuple((runtime or {}).get(k) for k in keys) 

638 

639 

640def _serve_flags(runtime): 

641 """The serve arguments minus --host: a port move is not a config change.""" 

642 args = list((runtime or {}).get("serve_args") or []) 

643 if "--host" in args: 

644 i = args.index("--host") 

645 del args[i : i + 2] 

646 return args 

647 

648 

649def tool_fingerprint(*paths): 

650 """Hash of the benchmark's own source: pass what decides the numbers, not plumbing.""" 

651 h = hashlib.sha256() 

652 here = os.path.dirname(os.path.abspath(__file__)) 

653 # Sort by basename: callers mix absolute paths and bare names. 

654 for name in sorted(paths, key=lambda p: (os.path.basename(p), p)): 

655 try: 

656 with open(os.path.join(here, name), "rb") as f: 

657 # autocrlf makes one commit CRLF on Windows and LF elsewhere. 

658 h.update(f.read().replace(b"\r\n", b"\n")) 

659 except OSError: 

660 return None 

661 return h.hexdigest()[:16] 

662 

663 

664def busy_lanes(registry_path=None): 

665 """Which other endpoints were serving while this ran: results shift with them.""" 

666 try: 

667 from orchestrant.benchmark.openai_api import load_backends 

668 

669 backends, _ = load_backends(registry_path) 

670 except Exception: 

671 return None 

672 # Never probe:false (paid) hosts; in parallel, as each refused connect waits out its timeout. 

673 urls = { 

674 name: entry["base_url"] 

675 for name, entry in sorted(backends.items()) 

676 if entry.get("base_url") and entry.get("probe", True) 

677 } 

678 

679 def answers(url): 

680 try: 

681 with urllib.request.urlopen(f"{url.rstrip('/')}/v1/models", timeout=2): # nosec B310 -- endpoint URL from config 

682 return True 

683 except Exception: 

684 return False 

685 

686 with ThreadPoolExecutor(max_workers=max(1, len(urls))) as pool: 

687 up = dict(zip(urls, pool.map(answers, urls.values()), strict=True)) 

688 return [name for name in urls if up[name]] 

689 

690 

691def energy_proxy(): 

692 """CPU-seconds consumed, recorded as a PROXY for energy, never as a measurement.""" 

693 try: 

694 import resource 

695 

696 me = resource.getrusage(resource.RUSAGE_SELF) # ty: ignore[unresolved-attribute] 

697 kids = resource.getrusage(resource.RUSAGE_CHILDREN) # ty: ignore[unresolved-attribute] 

698 return { 

699 "cpu_seconds_self": round(me.ru_utime + me.ru_stime, 3), 

700 "cpu_seconds_children": round(kids.ru_utime + kids.ru_stime, 3), 

701 "note": ( 

702 "CPU time, not joules. This host exposes no power rail " 

703 "(no RAPL on aarch64, no battery counter through WSL2), and " 

704 "this number says nothing about NPU or GPU draw." 

705 ), 

706 } 

707 except Exception: 

708 return None 

709 

710 

711# Windows 11's power-mode overlay moves CPU clocks, so a CPU-lane number needs it. 

712_POWER_OVERLAYS = { 

713 "961cc777-2547-4f9d-8174-7d86181b8a7a": "best power efficiency", 

714 "00000000-0000-0000-0000-000000000000": "balanced", 

715 "3af9b8d9-7c97-431d-ad78-34a8bfea439f": "better performance", 

716 "ded574b5-45a0-4f42-8737-46345c09c238": "best performance",