Coverage for orchestrant/benchmark/__main__.py: 38%

1"""``orchestrant-bench`` -- the measurement commands. 

2 

3One entry point over the CLIs the runner ships: the endpoint speed benchmark, 

4the lane/batching probes, the result summariser, the server-contract check, 

5the lane-runtime snapshot and the decode-at-depth trace. Each module 

6keeps its own argparse surface; this dispatcher only routes. 

7""" 

8 

9from __future__ import annotations 

10 

11import sys 

12 

13from orchestrant.benchmark import ( 

14 contract, 

15 depth, 

16 lanes, 

17 openai_api, 

18 provenance, 

19 report, 

20) 

21from orchestrant.benchmark.client import utf8_stdio 

22 

23 

24COMMANDS = { 

25 "speed": openai_api.main, 

26 "lanes": lanes.main, 

27 "report": report.main, 

28 "contract": contract.main, 

29 "runtimes": provenance.main, 

30 "depth": depth.main, 

31} 

32 

33USAGE = """usage: orchestrant-bench <command> [options] 

34 

35commands: 

36 speed throughput, TTFT, decode and correctness against an endpoint 

37 lanes streaming, batching and multi-lane additivity probes 

38 report summarise result files and build a viewer manifest 

39 contract re-check the server behaviours the tooling assumes; --diff two runs 

40 runtimes snapshot each lane's build, flags and model files for WSL2 reports 

41 depth decode rate per window of one long reply after a deep context 

42""" 

43 

44 

45def main(argv=None): 

46 utf8_stdio() 

47 args = list(sys.argv[1:] if argv is None else argv) 

48 if not args or args[0] in ("-h", "--help"): 

49 print(USAGE, end="" if args else "", file=sys.stdout if args else sys.stderr) 

50 return 0 if args else 2 

51 command = args.pop(0) 

52 handler = COMMANDS.get(command) 

53 if handler is None: 

54 print(USAGE, end="", file=sys.stderr) 

55 return 2 

56 sys.argv = [f"orchestrant-bench {command}", *args] 

57 return handler() 

58 

59 

60if __name__ == "__main__": 

61 raise SystemExit(main())