OrchestrANT Documentation

OrchestrANT Documentation#

Contents

Testing & Coverage#

Coverage Reports#

Test Results#

View detailed test results for the image’s Python, GIL and free-threaded:

The test reports include:

  • Pass/fail status for each test

  • Test duration

  • Error messages and tracebacks

  • Test summary statistics

Latest Markdown Test Report#

filepath

function

passed

skipped

SUBTOTAL

tests/integration/test_dummy.py

test_pipeline

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestSseDialects.test_ollama_style_spaced_data_prefix

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestSseDialects.test_geniex_style_unspaced_data_prefix

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestSseDialects.test_reported_usage_is_not_flagged_as_estimated

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestSseDialects.test_a_rate_is_read_only_from_a_window_over_the_floor

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestSseDialects.test_a_paced_stream_is_rated_over_its_window

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelDetection.test_prefers_openai_models_endpoint

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelDetection.test_falls_back_to_ollama_api_tags

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelDetection.test_unknown_when_nothing_answers

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelDetection.test_several_listed_models_are_refused_not_guessed

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelDetection.test_listing_returns_every_id

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestEnvVarCompat.test_legacy_ollama_base_url_still_honoured

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestEnvVarCompat.test_new_name_wins_over_legacy

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestEnvVarCompat.test_default_is_ollama_localhost

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelResolution.test_an_explicit_model_wins

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelResolution.test_the_backend_default_is_used_next

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelResolution.test_otherwise_the_endpoint_is_asked

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelResolution.test_the_probe_carries_the_entry_so_a_hosted_endpoint_authenticates

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelResolution.test_a_probe_false_backend_is_never_asked

1

0

1

tests/unit/benchmark/test_backend_compat.py

TestModelResolution.test_a_probe_false_backend_with_a_model_is_fine

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestShippedRegistry.test_repo_registry_parses_and_lists_both_backends

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestShippedRegistry.test_every_entry_has_a_base_url

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestShippedRegistry.test_no_entry_carries_an_unknown_field

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestShippedRegistry.test_no_entry_holds_a_key_rather_than_a_variable_name

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestShippedRegistry.test_optional_fields_have_the_documented_types

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestShippedRegistry.test_a_probe_false_backend_names_its_model

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestBackendEntry.test_named_entry_comes_back_whole

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestBackendEntry.test_an_explicit_url_alone_carries_no_entry

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestBackendEntry.test_the_backend_wins_over_an_explicit_url

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestBackendEntry.test_the_default_backend_supplies_the_entry

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestBackendEntry.test_an_unknown_backend_fails_loudly

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestBackendEntry.test_mutating_the_entry_does_not_edit_the_registry

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestResolutionOrder.test_named_backend

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestResolutionOrder.test_default_when_nothing_given

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestResolutionOrder.test_explicit_url_beats_named_backend

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestResolutionOrder.test_unknown_backend_fails_loudly_and_lists_options

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestResolutionOrder.test_missing_registry_still_resolves_an_explicit_url

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestResolutionOrder.test_malformed_registry_does_not_raise

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestEnvironmentPrecedence.test_env_beats_named_backend

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestEnvironmentPrecedence.test_legacy_env_name_also_beats_backend

1

0

1

tests/unit/benchmark/test_backends_registry.py

TestEnvironmentPrecedence.test_env_still_yields_the_backend_default_model

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSplitAnswer.test_a_think_block_that_never_closed_is_all_thinking

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSplitAnswer.test_a_closed_block_leaves_the_answer

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSplitAnswer.test_separate_reasoning_counts_as_thinking

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAccounting.test_cut_inside_think_is_all_thinking_and_unanswered

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAccounting.test_an_answer_cut_at_the_cap_is_not_a_finished_answer

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAccounting.test_no_finish_reason_at_the_budget_is_a_cut

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAccounting.test_a_plain_finished_answer

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAccounting.test_a_cut_reply_with_no_marker_records_no_share_and_why

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAShareTheReplyCannotSay.test_a_cut_reply_with_no_marker_and_no_reasoning_is_unknown

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAShareTheReplyCannotSay.test_a_finished_reply_with_no_marker_is_still_no_thinking

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAShareTheReplyCannotSay.test_a_cut_reply_that_shows_its_thinking_keeps_its_share

3

0

3

tests/unit/benchmark/test_bench_answers.py

TestAShareTheReplyCannotSay.test_an_empty_reply_has_no_share_and_nothing_to_explain

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAnOlderReportsShareOfACutReply.test_the_9b_distills_cut_rows_read_as_unknown

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAnOlderReportsShareOfACutReply.test_a_speed_row_cut_at_the_cap_reads_as_unknown

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAnOlderReportsShareOfACutReply.test_a_coding_row_that_passed_at_the_deadline_reads_as_unknown

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAnOlderReportsShareOfACutReply.test_a_finished_row_keeps_its_zero

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAnOlderReportsShareOfACutReply.test_a_row_that_says_reads_as_written

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestReadStream.test_first_answer_comes_after_the_thinking

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestReadStream.test_reasoning_content_is_output_not_prefill

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestReadStream.test_usage_after_the_last_choice_is_kept

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestDecodeFields.test_a_burst_has_no_rate_and_says_why

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestDecodeFields.test_the_shortest_real_windows_keep_their_rates

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestDecodeFields.test_the_floor_lies_between_the_burst_and_the_real_windows

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestDecodeFields.test_a_one_token_reply_decoded_nothing

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestDecodeFields.test_an_unstreamed_reply_has_no_window

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestRowDecodeRate.test_an_older_reports_burst_has_no_rate

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestRowDecodeRate.test_an_older_reports_shortest_real_window_keeps_its_rate

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestRowDecodeRate.test_a_row_decode_fields_wrote_reads_as_written

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestRowDecodeRate.test_a_rate_without_a_token_count_is_kept

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSummary.test_cut_rows_are_counted_and_left_out_of_time_to_answer

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSummary.test_an_older_report_s_unclosed_think_reads_as_all_thinking

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSummary.test_the_unknown_shares_are_counted_not_averaged

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSummary.test_a_run_whose_every_share_is_unknown_says_so

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestSummary.test_answer_seconds_only_for_answered_rows

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestUtf8Stdio.test_a_cp1252_stream_prints_arrows_instead_of_raising

1

0

1

tests/unit/benchmark/test_bench_answers.py

TestAnswerCell.test_a_cut_row_says_so_instead_of_none

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestSingleRun.test_explicit_label_wins

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestSingleRun.test_model_is_the_label_when_none_given

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestSingleRun.test_a_backend_pinned_model_is_used

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestSingleRun.test_label_never_ends_up_none

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestSingleRun.test_label_never_none_even_with_no_backend_at_all

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCompareFile.test_reads_every_entry

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCompareFile.test_entries_without_label_or_model_still_get_a_label

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCompareFile.test_explicit_base_url_overrides_the_backend

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCompareFile.test_a_non_list_file_fails_loudly

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestWriteReport.test_writes_the_shared_envelope

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestWriteReport.test_the_one_served_model_is_handed_to_collect

4

0

4

tests/unit/benchmark/test_bench_cli.py

TestWriteReport.test_the_write_is_atomic

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestWriteReport.test_no_plumbing_is_in_the_fingerprint

8

0

8

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_content_type_by_default

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_api_key_env_becomes_a_bearer_header

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_a_missing_variable_fails_loudly_and_names_it

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_an_empty_variable_is_treated_as_missing

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_building_the_header_does_not_stash_the_key_in_the_entry

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_entry_headers_are_merged

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRequestHeaders.test_api_key_env_wins_over_a_literal_authorization_header

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestEntryConfig.test_reports_the_extras

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestEntryConfig.test_header_values_never_reach_a_report

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestEntryConfig.test_the_key_variable_name_is_kept_but_not_read

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestEntryConfig.test_probe_defaults_to_true

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_an_ordinary_command_is_kept_whole

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_a_flag_named_for_a_credential_loses_its_value

5

0

5

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_a_switch_does_not_take_the_next_flag_with_it

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_a_value_shaped_like_a_key_is_redacted_under_any_flag

7

0

7

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_a_file_named_like_a_prefix_is_not_a_key

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_write_report_records_the_redacted_command_line

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestRedactArgv.test_compare_neither_needs_it_nor_reads_it

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_the_body_is_sent_as_json

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_request_extra_is_merged_in

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_an_explicit_key_wins_over_request_extra

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_auth_travels_with_the_request

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_stream_sets_the_body_flag

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_a_caller_can_still_turn_streaming_off_in_the_body

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_json_parses_the_response

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_lines_decodes_and_strips

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_the_deadline_abandons_the_stream_and_says_so

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_no_deadline_reads_the_whole_stream

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_the_context_manager_closes_the_socket

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestPostJson.test_the_timeout_reaches_urlopen

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestHttpErrorDetail.test_an_http_error_gives_its_status_and_body

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestHttpErrorDetail.test_the_body_is_cut_to_the_limit

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestHttpErrorDetail.test_undecodable_bytes_do_not_cost_the_status

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestHttpErrorDetail.test_an_error_without_a_body_still_has_its_status

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestHttpErrorDetail.test_any_other_failure_has_no_detail

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestLabelCollisions.test_colliding_derived_labels_get_the_backend_appended

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestLabelCollisions.test_a_unique_label_keeps_the_bare_model_id

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestLabelCollisions.test_colliding_explicit_labels_are_refused

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestLabelCollisions.test_the_same_backend_twice_is_refused_rather_than_silently_merged

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCandidateEntries.test_three_tuples_stay_the_default

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCandidateEntries.test_asking_for_entries_yields_four_tuples

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCandidateEntries.test_the_single_run_path_carries_an_entry_too

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCandidateEntries.test_a_comment_only_element_is_not_a_candidate

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestCandidateEntries.test_a_commented_candidate_is_still_a_candidate

1

0

1

tests/unit/benchmark/test_bench_cli.py

TestShippedExampleCandidates.test_the_example_file_resolves

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestFiller.test_is_deterministic_and_roughly_sized

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestFiller.test_a_different_seed_differs

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_max_tokens_honoured_needs_the_count_and_the_reason

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_a_transport_error_is_error_not_no

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_power_mode_is_understood_only_if_nonsense_is_refused

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_power_mode_leaves_the_lane_as_launched

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_temperature0_draws_are_never_back_to_back

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_temperature0_read_as_unset_is_named

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_identical_repeat_is_judged_only_on_a_reproducible_setting

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_thinking_in_a_separate_field_counts

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_a_fast_cold_prefill_cannot_show_a_cache

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_a_server_error_mentioning_the_context_is_not_clean

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_completions_stop_needs_the_sequence_to_get_there

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_overflow_is_skipped_unless_asked

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_an_overflow_error_naming_the_context_is_clean

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_the_three_cache_answers_share_one_measurement

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_no_cache_at_all

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_a_cached_repeat_must_keep_its_prompt_size

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_a_stop_string_that_never_came_up_is_inconclusive

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestVerdicts.test_tool_call_must_arrive_as_tool_calls

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_a_spent_budget_is_no_cap

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_a_length_stop_short_of_the_budget_is_a_cap_and_says_where

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_a_length_stop_at_any_count_short_of_the_budget_is_a_cap

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_a_round_number_reported_as_a_finish_is_a_cap

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_a_reply_that_finished_on_its_own_is_inconclusive

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_no_completion_count_is_inconclusive

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_more_than_asked_is_named

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestOutputCap.test_the_long_reply_gets_a_long_timeout

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestResponseFormat.test_the_schema_object_is_honoured

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestResponseFormat.test_prose_is_ignored

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestResponseFormat.test_json_of_another_shape_is_a_json_mode_without_the_schema

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestResponseFormat.test_a_fenced_object_was_steered_not_constrained

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestResponseFormat.test_a_4xx_is_refused_and_a_5xx_is_an_error

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestResponseFormat.test_thinking_is_stripped_and_an_unclosed_block_is_no_answer

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_without_a_default_only_the_framing_shows

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_a_default_that_an_explicit_message_replaces

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_the_requests_alternate_and_never_repeat

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_a_failed_throwaway_does_not_stop_the_measurement

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_a_system_message_that_costs_nothing_is_inconclusive

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_disagreeing_no_system_counts_are_inconclusive

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_missing_counts_are_inconclusive

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_the_self_report_is_recorded_and_never_votes

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestBundleSystemPrompt.test_a_failed_measurement_is_an_error_naming_the_request

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTheNewChecksAreRegistered.test_every_new_check_is_in_the_run_order

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestDiff.test_names_every_moved_answer_in_check_order

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestDiff.test_a_check_only_one_side_ran_is_a_change

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_the_default_prefix_keeps_the_old_timeout

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_a_long_prefix_gets_time_for_the_slowest_lane_measured

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_a_longer_timeout_asked_for_wins

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_every_prefix_request_gets_the_scaled_timeout

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_the_overflow_prompt_is_scaled_the_same_way

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_the_scaled_timeout_reaches_the_request

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_the_output_cap_keeps_its_floor_and_takes_a_longer_timeout

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_an_ordinary_request_takes_the_timeout

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestTimeouts.test_the_report_records_them

1

0

1

tests/unit/benchmark/test_bench_contract.py

TestModelDetection.test_the_model_is_asked_of_the_lane_being_probed

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestPrompt.test_it_is_the_scratch_scripts_prompt

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestPrompt.test_the_8000_token_prompt_is_pinned

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestPrompt.test_no_context_is_the_bare_question

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_a_1024_token_reply_reads_as_the_tracked_trace_does

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_a_short_reply_ends_where_the_ollama_trace_did

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_the_prefill_never_enters_a_window

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_a_tail_under_a_quarter_window_is_dropped

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_the_window_size_is_a_parameter

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_a_window_that_arrived_in_one_burst_has_no_rate

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestWindows.test_nothing_to_window

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_the_request_is_the_scratch_scripts

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_stamps_are_seconds_from_the_request

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_thinking_is_output_too

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_a_failed_request_is_recorded_not_raised

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_what_arrived_before_a_timeout_is_kept

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_a_server_error_inside_the_stream_is_an_error

2

0

2

tests/unit/benchmark/test_bench_depth.py

TestMeasure.test_a_reply_with_no_generated_token_is_an_error

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestRow.test_the_scratch_scripts_fields_and_the_new_ones

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestRow.test_no_tokens_is_a_null_ttft_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestRow.test_a_tracked_trace_has_no_field_a_new_row_lacks

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_writes_the_envelope_with_provenance

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_the_config_says_what_was_asked

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_the_rest_is_the_host_load_window

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_a_failed_trace_is_written_and_exits_1

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_an_error_inside_the_stream_exits_1

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_the_report_is_written_before_the_summary_prints

1

0

1

tests/unit/benchmark/test_bench_depth.py

TestMain.test_orchestrant_bench_routes_it

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestInterpolate.test_linear_between_published_samples

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestInterpolate.test_outside_the_sampled_range_is_unknown_not_extrapolated

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestEnergyBetween.test_joules_over_a_window_that_straddles_updates

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestEnergyBetween.test_a_window_the_meter_has_not_covered_is_none

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestEnergyBetween.test_an_unavailable_meter_reports_nothing

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestRequestEnergy.test_gross_net_and_per_token

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestRequestEnergy.test_no_meter_no_fields_rather_than_zeros

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestRequestEnergy.test_a_disabled_meter_adds_no_fields

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestRequestEnergy.test_net_energy_never_goes_negative

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestSamples.test_a_torn_read_is_dropped_and_the_next_poll_recovers

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestSamples.test_a_counter_reset_starts_the_curve_again

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestBaselines.test_a_zero_second_idle_window_is_no_baseline_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestBaselines.test_rows_are_netted_against_the_mean_of_before_and_after

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestBaselines.test_one_baseline_cannot_vouch_for_itself

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestBaselines.test_drift_between_the_baselines_is_reported

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestAvailability.test_outside_windows_the_meter_says_why

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestAvailability.test_scope_names_what_is_not_metered

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_a_direct_lane_is_not_the_gateway

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_an_alias_reaches_its_lane_then_its_overflow_lane

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_localhost_and_a_trailing_slash_are_the_same_gateway

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_a_raw_route_reaches_only_its_lane

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_raw_routes_switched_off_route_nothing

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_an_unrouted_alias_is_named_not_guessed

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_a_lane_pointing_back_at_the_gateway_is_dropped

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_without_its_primary_an_alias_names_no_lane_behind_it

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_a_malformed_serving_block_is_an_error_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_a_registry_without_a_serving_block_has_no_gateway

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRoute.test_the_registry_sha_is_the_one_the_renderer_puts_in_gateway_info

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestAgreesWithTheHub.test_every_alias_reaches_the_lanes_the_renderer_routes

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestAgreesWithTheHub.test_every_registry_entry_at_the_gateway_names_a_lane

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestAgreesWithTheHub.test_gateway_info_carries_every_key_a_report_keeps

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestAgreesWithTheHub.test_the_plugin_sets_the_headers_note_reply_counts

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestServed.test_a_reply_is_counted_by_the_lane_it_names

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestServed.test_a_direct_lane_reply_counts_nothing

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestServed.test_a_header_that_is_not_a_string_is_no_evidence

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestThroughPostJson.test_a_gateway_reply_is_counted

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestThroughPostJson.test_a_refusal_is_counted_and_still_raised

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestThroughPostJson.test_a_direct_reply_leaves_no_trace

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestInfo.test_only_the_known_shas_are_kept

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestInfo.test_a_gateway_that_is_down_is_a_recorded_gap

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestInfo.test_a_body_that_is_not_an_object_is_a_recorded_gap

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestGatewayBlock.test_a_direct_lane_gets_none_and_asks_nothing

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestGatewayBlock.test_the_gateway_block_names_the_lanes_and_what_served

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestGatewayBlock.test_a_gateway_rendered_from_another_registry_says_so

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestGatewayBlock.test_an_unreachable_gateway_leaves_the_match_unknown

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRuntimeInfo.test_a_gateway_alias_answers_with_its_lanes_runtime

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestRuntimeInfo.test_a_direct_lane_is_looked_up_as_before

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCollect.test_a_gateway_report_carries_the_gateway_block

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCollect.test_a_direct_report_carries_a_null_gateway

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_two_direct_runs_say_nothing

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_the_same_gateway_twice_says_nothing

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_a_gateway_run_against_a_direct_one_is_named

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_a_changed_gateway_is_named

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_a_run_that_overflowed_is_told_from_one_that_did_not

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_compare_carries_the_gateway_notes

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_a_gateway_run_against_a_direct_one_blames_no_pull

1

0

1

tests/unit/benchmark/test_bench_gateway.py

TestCompare.test_two_gateway_runs_still_compare_their_model_lists

1

0

1

tests/unit/benchmark/test_bench_gateway.py

test_registry_sha256_is_canonical_json

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestCollectGpuInfo.test_records_vendor_name_vram_and_bandwidth

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestCollectGpuInfo.test_no_gpu_is_an_empty_dict_not_a_zero_row

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestSampleGpuResources.test_reports_utilization_memory_and_power

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestSampleGpuResources.test_no_probe_returns_nothing

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestSampleGpuResources.test_a_broken_read_does_not_raise

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestSampleResourcesMerge.test_gpu_fields_ride_along_with_cpu_and_ram

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestSampleResourcesMerge.test_a_flaky_glances_falls_back_to_psutil

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestSampleResourcesMerge.test_a_refused_glances_is_asked_once_per_run

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestAvgOptional.test_averages_when_both_samples_carry_the_metric

1

0

1

tests/unit/benchmark/test_bench_gpu.py

TestAvgOptional.test_absent_metric_is_none_not_zero

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestPort.test_loopback_urls_have_a_local_port

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestPort.test_a_remote_url_is_not_this_host

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestLaneProcess.test_a_remote_endpoint_is_unavailable_with_a_reason

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestLaneProcess.test_finds_the_process_listening_on_a_local_port

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestListensHere.test_nothing_on_the_port

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestListensHere.test_a_listener_whose_pid_is_hidden

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestListensHere.test_a_visible_listener

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestListensHere.test_a_failed_lookup_is_unknown

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestListensHere.test_a_remote_url_is_never_looked_up

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestRemoteLaneIsNotMetered.test_this_hosts_rails_are_not_recorded_for_another_host

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestLaneThatDisappears.test_a_gone_lane_is_unknown_not_zero

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestSummaryLines.test_energy_per_token_is_a_ratio_of_sums

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestSummaryLines.test_other_load_is_named_beside_the_lane

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestSummaryLines.test_no_metered_rows_no_energy_line

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestRequestBracket.test_other_cores_is_system_load_minus_the_lane

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestRequestBracket.test_no_other_cores_without_a_local_lane

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestWindow.test_lane_cpu_seconds_are_differenced_over_the_window

1

0

1

tests/unit/benchmark/test_bench_hostload.py

TestWindow.test_system_busy_share_is_a_percentage

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestStreamOnce.test_parses_sse_without_space_after_data

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestStreamOnce.test_unreachable_endpoint_reports_error_not_raise

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestBatchingVerdict.test_detects_serialised_server

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestBatchingVerdict.test_detects_overlapping_server

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneSpecParsing.test_parses_name_url_model

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneSpecParsing.test_strips_trailing_slash

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneSpecParsing.test_model_name_may_contain_commas_after_the_marker

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneSpecParsing.test_rejects_malformed_spec

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneResolution.test_bare_backend_name_resolves_from_registry

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneResolution.test_full_spec_still_works

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneResolution.test_unknown_name_is_rejected_with_the_known_list

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestLaneResolution.test_backend_without_a_default_model_explains_itself

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestEmptyReplyHandling.test_seconds_formatter_tolerates_none

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestEmptyReplyHandling.test_batching_probe_is_inconclusive_rather_than_crashing

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestRegistrySeam.test_resolves_against_a_supplied_registry

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestRegistrySeam.test_unknown_name_lists_the_supplied_registry_not_the_shipped_one

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestReportEnvelope.test_one_row_per_lane_plus_the_aggregate

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestReportEnvelope.test_batching_row_carries_the_verdict

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestReportEnvelope.test_a_failed_batching_probe_is_still_a_row

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestReportEnvelope.test_no_rows_are_scores

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestReportEnvelope.test_main_writes_the_shared_envelope

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestEachLaneNamesItsRuntime.test_every_lane_row_carries_its_own_runtime

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestEachLaneNamesItsRuntime.test_a_run_from_before_the_field_has_null_runtimes

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestEachLaneNamesItsRuntime.test_runtimes_are_taken_before_anything_is_measured

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestPhasesAreCold.test_every_request_gets_its_own_prompt

1

0

1

tests/unit/benchmark/test_bench_lanes.py

TestAggregateIsDelivered.test_the_report_judges_delivered_and_keeps_the_sum

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCollect.test_records_the_fields_a_rerun_needs

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCollect.test_names_missing_fields_instead_of_omitting_them

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCollect.test_unreachable_server_is_recorded_as_null_not_an_exception

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCollect.test_extra_fields_are_merged

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCollect.test_the_served_model_reaches_runtime_info

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestToolFingerprint.test_hashes_the_benchmark_source

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestToolFingerprint.test_same_files_same_hash_regardless_of_order

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestToolFingerprint.test_different_files_differ

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestToolFingerprint.test_missing_file_yields_none

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCompare.test_two_identical_clean_runs_report_nothing

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCompare.test_a_dirty_run_is_flagged_even_against_itself

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCompare.test_a_changed_grader_is_called_out_first

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCompare.test_different_served_models_are_flagged

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCompare.test_a_dirty_tree_is_flagged

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestCompare.test_a_moved_repository_is_flagged

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntime.test_parses_all_three_version_lines

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntime.test_unrecognised_output_parses_to_nothing

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntime.test_a_runtime_upgrade_is_named_on_compare

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntime.test_an_identical_runtime_says_nothing

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntime.test_a_runtime_on_one_side_only_is_flagged

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntime.test_changed_serve_flags_are_named_but_a_port_move_is_not

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntimeInfo.test_the_lane_process_is_the_best_evidence

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntimeInfo.test_an_unreachable_loopback_url_is_not_attributed_to_geniex

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_a_gguf_is_named_by_size_and_a_hash_of_its_first_mib

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_two_quants_share_a_head_and_differ_in_the_sample

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_an_edit_between_the_windows_is_unseen

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_the_variant_matches_ignoring_case

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_an_id_naming_no_variant_of_several_is_not_guessed

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_a_qairt_bundle_records_its_config_hash_and_sampler

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_a_config_edited_in_place_no_longer_matches_the_manifest

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_a_bundle_config_naming_no_weights_is_an_error

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_the_cache_is_never_written

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_a_gap_is_an_error_never_an_exception

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_from_wsl_the_windows_cache_is_searched_first

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_on_windows_the_users_own_cache_is_the_one

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_the_lane_process_resolves_the_model_it_serves

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFiles.test_the_installed_binary_guess_names_files_and_drivers

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDriverVersions.test_reads_the_npu_and_gpu_drivers_from_the_registry

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDriverVersions.test_an_unreadable_class_is_a_reason_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDriverVersions.test_off_windows_they_are_null_with_the_reason

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_it_stands_in_for_an_invisible_lane_process

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_a_stale_snapshot_keeps_its_data_but_loses_verified

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_an_upgrade_since_the_snapshot_loses_verified

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_a_remote_url_never_matches_by_port

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_an_ambiguous_port_matches_nothing

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_a_missing_or_broken_file_is_no_snapshot

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_an_entry_the_host_could_not_attribute_is_no_evidence

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_another_model_than_the_snapshots_is_resolved_again

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_written_on_the_host_and_read_back

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_a_lane_whose_runtime_raises_is_an_error_entry

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestLaneRuntimeFile.test_the_command_writes_the_file

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFilesNotes.test_a_replaced_file_behind_the_same_id_is_named

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFilesNotes.test_an_edited_bundle_config_names_the_sampler

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFilesNotes.test_different_ids_or_unrecorded_files_say_nothing

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFilesNotes.test_a_file_one_side_could_not_read_is_not_a_change

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFilesNotes.test_an_edited_extensions_file_is_named

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestModelFilesNotes.test_compare_reports_them

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestRuntimesCommand.test_orchestrant_bench_routes_it_to_the_snapshot_writer

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestBusyLanes.test_a_probe_false_backend_is_never_asked

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestTemperatureAndSeed.test_defaults_are_explicit_nulls

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestTemperatureAndSeed.test_values_are_recorded

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestTemperatureAndSeed.test_positional_callers_still_work

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestTemperatureAndSeed.test_a_temperature_change_is_flagged_on_compare

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestFingerprintLineEndings.test_crlf_and_lf_checkouts_hash_the_same

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestSourceChangedDuringRun.test_a_mid_run_edit_is_recorded_and_named

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestSourceChangedDuringRun.test_an_unchanged_source_is_recorded_as_checked

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestSourceChangedDuringRun.test_no_start_hash_records_nothing

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestSourceChangedDuringRun.test_the_speed_runner_path_records_the_start_load

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestSourceChangedDuringRun.test_the_speed_runner_takes_a_run_start_record

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_two_identical_answers_are_deterministic

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_the_draws_are_never_back_to_back

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_two_different_answers_are_not

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_a_failed_probe_is_null_not_an_exception

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_the_probe_leaves_the_model_real_choices

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_a_reply_without_content_is_a_failed_probe

1

0

1

tests/unit/benchmark/test_bench_provenance.py

TestDeterminismProbe.test_known_deterministic_needs_a_positive_probe

1

0

1

tests/unit/benchmark/test_bench_report.py

TestSummarise.test_counts_only_successful_results

1

0

1

tests/unit/benchmark/test_bench_report.py

TestSummarise.test_a_file_with_no_successes_returns_none

1

0

1

tests/unit/benchmark/test_bench_report.py

TestSummarise.test_missing_ttft_is_none_not_zero

1

0

1

tests/unit/benchmark/test_bench_report.py

TestSummarise.test_gpu_utilization_averages_over_the_prompts_that_carry_it

1

0

1

tests/unit/benchmark/test_bench_report.py

TestSummarise.test_reads_the_newer_envelope_too

1

0

1

tests/unit/benchmark/test_bench_report.py

TestResultFileSelection.test_skips_generated_files

1

0

1

tests/unit/benchmark/test_bench_report.py

TestResultFileSelection.test_an_empty_directory_is_not_an_error

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifest.test_lists_every_result_file

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifest.test_takes_hardware_from_the_first_file_that_has_it

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifest.test_accepts_provenance_as_hardware

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifest.test_a_hardware_block_beats_a_provenance_that_sorts_first

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifest.test_survives_a_manifest_already_in_the_directory

1

0

1

tests/unit/benchmark/test_bench_report.py

TestComparisonTable.test_one_row_per_result_file

1

0

1

tests/unit/benchmark/test_bench_report.py

TestComparisonTable.test_files_without_successes_are_skipped_not_crashed_on

1

0

1

tests/unit/benchmark/test_bench_report.py

TestScoredRowsAreCounts.test_a_scored_row_needs_integer_passed_and_total

1

0

1

tests/unit/benchmark/test_bench_report.py

TestScoredRowsAreCounts.test_turn_growth_has_no_scored_rows

1

0

1

tests/unit/benchmark/test_bench_report.py

TestScoredRowsAreCounts.test_embeddings_semantic_score_is_not_a_scored_row

1

0

1

tests/unit/benchmark/test_bench_report.py

TestScoredRowsAreCounts.test_booleans_are_not_counts

1

0

1

tests/unit/benchmark/test_bench_report.py

TestScoredRowsAreCounts.test_a_mixed_envelope_keeps_only_the_real_scores

1

0

1

tests/unit/benchmark/test_bench_report.py

TestReportKind.test_lanes_envelope_is_labelled_by_its_benchmark

1

0

1

tests/unit/benchmark/test_bench_report.py

TestReportKind.test_legacy_shape_is_throughput

1

0

1

tests/unit/benchmark/test_bench_report.py

TestReportKind.test_a_json_that_is_no_report_is_unknown_and_warned

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifestCarriesWhatTheViewerShows.test_a_speed_report_carries_runtime_energy_and_threads

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifestCarriesWhatTheViewerShows.test_a_legacy_report_is_dated_by_its_own_timestamp

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifestCarriesWhatTheViewerShows.test_a_contract_report_keeps_its_checks_and_runtime

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifestCarriesWhatTheViewerShows.test_an_older_report_carries_nones_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_report.py

TestManifestCarriesWhatTheViewerShows.test_the_legacy_url_stands_in_for_a_missing_provenance

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_energy_and_load_match_the_published_table

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_the_answer_run_matches_the_published_comparison

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_the_contract_grid_shows_what_the_upgrade_changed

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_runtime_rows_name_the_build_and_how_it_was_seen

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_the_comparison_row_is_the_runners_speed_summary

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_a_scored_reports_ttft_comes_from_its_cases

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheViewerReadsTheTrackedRun.test_the_hardware_card_is_the_hosts_not_a_provenance

1

0

1

tests/unit/benchmark/test_bench_report.py

TestAnswerCarriesItsCount.test_a_run_with_cut_replies_says_how_many_answered

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheManifestSplitsTheProbe.test_an_old_speed_report_reaches_the_viewer_split

1

0

1

tests/unit/benchmark/test_bench_report.py

TestTheManifestSplitsTheProbe.test_a_report_without_a_probe_still_has_none

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadSnapshot.test_other_cores_is_busy_minus_the_lane

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadSnapshot.test_without_a_lane_every_busy_core_is_other

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadSnapshot.test_a_lane_on_another_host_leaves_other_load_unknown

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadSnapshot.test_a_lane_that_restarted_is_unknown_not_zero

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadSnapshot.test_the_lane_never_counts_below_zero_other_load

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadSnapshot.test_without_psutil_nothing_is_invented

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadLine.test_a_busy_host_is_warned_about

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadLine.test_a_quiet_host_is_not

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadLine.test_an_unknown_load_says_why

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestRunStart.test_records_the_time_the_hash_and_the_load

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestRunStart.test_a_failed_reading_never_costs_the_run

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestWriteReportCarriesTheStart.test_load_start_time_and_an_unchanged_source

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestWriteReportCarriesTheStart.test_a_mid_run_edit_is_caught

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestWriteReportCarriesTheStart.test_another_file_set_is_not_read_as_an_edit

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestWriteReportCarriesTheStart.test_without_a_start_the_fields_are_explicit_nulls

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_runs_under_different_load_are_named

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_the_spread_of_two_quiet_runs_is_not

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_exactly_the_threshold_apart_is_not_more_than_it

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_a_busy_host_gets_the_stronger_note

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_both_busy_is_said_even_at_equal_load

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_a_difference_needs_both_sides

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_a_busy_run_is_named_against_a_baseline_without_the_record

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_a_malformed_record_is_ignored_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestLoadNotes.test_load_is_not_liveness

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestContractDiffIsNotGated.test_a_busy_host_is_named_and_nothing_refused

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestContractDiffIsNotGated.test_a_moved_answer_still_exits_1

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestFingerprintScope.test_the_probe_is_re_exported_unchanged

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestFingerprintScope.test_the_probe_module_is_fingerprintable

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestFingerprintScope.test_the_hash_does_not_depend_on_how_a_path_is_spelled

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestFingerprintScope.test_a_changed_file_set_is_named_beside_the_source_note

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestFingerprintScope.test_collect_names_the_files_by_basename

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestToolMains.test_lanes

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestToolMains.test_contract

1

0

1

tests/unit/benchmark/test_bench_runstart.py

TestToolMains.test_contract_diff_takes_no_start

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestWilsonInterval.test_certainty_does_not_produce_a_zero_width_interval

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestWilsonInterval.test_zero_successes_is_handled

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestWilsonInterval.test_more_trials_narrow_the_interval

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestWilsonInterval.test_no_trials_yields_full_range

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestWilsonInterval.test_rejects_impossible_input

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestWilsonInterval.test_interval_always_contains_the_point_estimate

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestSeparability.test_the_case_that_motivated_this

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestSeparability.test_a_large_clear_difference_is_separable

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestSeparability.test_identical_scores_overlap

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestSeparability.test_note_names_the_overlap_explicitly

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestSeparability.test_note_confirms_a_real_difference

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestSeparability.test_note_handles_empty_input

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestFormatting.test_score_carries_its_interval

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestFormatting.test_no_trials_is_not_rendered_as_zero_percent

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestStatisticalPower.test_a_tiny_suite_can_only_prove_a_collapse

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestStatisticalPower.test_more_cases_detect_subtler_drops

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestStatisticalPower.test_the_measured_case_needs_about_27

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestStatisticalPower.test_a_suite_too_small_to_prove_anything_says_so

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestStatisticalPower.test_power_note_states_the_threshold

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestStatisticalPower.test_zero_trials_is_not_a_crash

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_six_one_way_flips_are_significant

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_three_one_way_flips_are_not

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_no_discordant_cases_is_no_evidence

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_two_sided_and_symmetric

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_never_exceeds_one

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_rejects_negative_counts

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedSignTest.test_the_case_the_overlap_rule_got_wrong

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestDiffInterval.test_contains_the_point_difference

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestDiffInterval.test_swapping_sides_mirrors_the_interval

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestDiffInterval.test_is_sharper_than_interval_overlap

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestDiffInterval.test_no_trials_yields_the_full_range

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestDiffInterval.test_identical_scores_straddle_zero

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedOutcomes.test_counts_bools

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedOutcomes.test_counts_repeat_pairs_by_rate

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedOutcomes.test_only_shared_cases_count_and_unmeasured_ones_are_skipped

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestTiers.test_six_flips_start_a_new_tier

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestTiers.test_three_flips_share_a_tier

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestTiers.test_tiers_chain_through_adjacent_rows

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestTiers.test_empty_ranking

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPower.test_six_flips_is_the_floor_at_five_percent

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPower.test_note_states_the_floor

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_the_tools_r3_shape_by_hand

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_it_is_wider_than_the_interval_it_replaces

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_one_draw_per_case_costs_only_the_small_sample_factor

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_every_attempt_agreeing_takes_the_case_as_the_unit

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_one_case_cannot_estimate_the_spread

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_a_design_effect_below_one_never_narrows_the_interval

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_the_interval_stays_inside_zero_and_one

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_unattempted_cases_are_skipped_and_nothing_is_none

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestClusteredRate.test_rejects_impossible_counts

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedDifference.test_identical_per_case_outcomes_are_zero_wide

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedDifference.test_six_of_27_flipping_by_hand

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedDifference.test_repeats_are_compared_as_per_case_rates

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedDifference.test_only_cases_measured_on_both_sides_count

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedDifference.test_the_interval_is_clamped

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedDifference.test_differences_that_cancel_are_exactly_zero

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPassHatK.test_the_tools_r3_shape_by_hand

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPassHatK.test_it_is_the_unbiased_estimator_not_the_plug_in

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPassHatK.test_k_of_one_is_the_mean_per_case_rate

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPassHatK.test_no_case_with_k_attempts_is_none

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_the_rejection_limits_are_the_sign_tests

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_power_matches_brute_force_enumeration

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_31_cases_catch_a_ten_point_drop_eight_to_eleven_percent

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_six_cases_by_hand

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_five_cases_can_never_regress

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_the_mde_is_where_power_crosses_eighty_percent

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_more_cases_and_fewer_back_flips_detect_less

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_the_note_says_where_the_back_flip_rate_came_from

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_the_default_is_a_floor_under_the_observed_rate

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestPairedPowerAndMde.test_a_back_flip_never_makes_the_test_look_sharper

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestNotes.test_the_clustered_note_only_where_repeats_disagree

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestNotes.test_the_paired_diff_note

1

0

1

tests/unit/benchmark/test_bench_stats.py

TestNotes.test_the_pass_k_note

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInWsl.test_microsofts_kernel_is_wsl

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInWsl.test_the_interop_handler_alone_is_enough

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInWsl.test_plain_linux_is_not

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInWsl.test_windows_itself_is_not

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInWsl.test_wsl1s_capitalised_kernel_is_wsl_too

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestFindPowershell.test_the_system32_path_comes_first

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestFindPowershell.test_path_when_c_is_mounted_elsewhere

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestFindPowershell.test_none_when_neither_exists

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestScript.test_the_port_and_the_window_are_integers

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestScript.test_no_localised_counter_path

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_busy_cores_from_the_idle_counter

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_lane_cores_over_the_span_of_its_own_samples

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_a_child_gone_mid_window_reads_no_lane_load_not_negative

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_idle_beyond_the_window_reads_zero_busy_not_negative

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_text_before_the_reading_is_ignored

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_nothing_listening_has_no_lane_fields

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_lane_samples_stamped_alike_have_no_lane_fields

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestParse.test_garbage_is_an_error_naming_why

8

0

8

tests/unit/benchmark/test_bench_winhost.py

TestMeasure.test_runs_windows_powershell_with_the_script_encoded

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestMeasure.test_every_failure_is_an_interop_error

3

0

3

tests/unit/benchmark/test_bench_winhost.py

TestMeasure.test_no_powershell_is_an_interop_error

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestMeasure.test_a_clixml_error_stream_is_read_as_its_text

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestErrorText.test_plain_text_is_kept

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestErrorText.test_xml_entities_and_escaped_characters_are_decoded

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestErrorText.test_clixml_without_an_error_record_is_empty

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_other_cores_is_the_windows_host_net_of_the_lane

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_one_scale_with_a_local_reading

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_a_lane_windows_does_not_see_either_is_unknown_not_zero

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_the_lane_never_counts_below_zero_other_load

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_a_lane_that_restarted_in_the_window_is_unknown

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_a_failed_reading_keeps_the_old_null_and_says_why

6

0

6

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_without_powershell_the_old_null_names_it

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_a_failure_does_not_add_a_second_window

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadSnapshotThroughInterop.test_an_early_failure_still_gives_the_fallback_its_window

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInteropIsNotTried.test_not_outside_wsl

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInteropIsNotTried.test_not_for_a_listener_here_whose_pid_is_hidden

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInteropIsNotTried.test_not_when_the_lookup_here_failed

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInteropIsNotTried.test_not_for_a_remote_lane

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInteropIsNotTried.test_not_without_a_lane

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestInteropIsNotTried.test_not_for_a_lane_visible_here

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadLine.test_a_windows_reading_is_named

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadLine.test_an_unknown_windows_reading_says_where_it_looked

1

0

1

tests/unit/benchmark/test_bench_winhost.py

TestLoadLine.test_a_local_reading_reads_as_before

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_plain_match

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_case_insensitive

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_ignores_thousands_separator

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_strips_markdown_emphasis

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_thinking_block_is_stripped

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_wrong_answer_after_correct_thinking_fails

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_word_boundary_prevents_substring_hit

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_decimal_expected_value_is_exact

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_multiple_accepted_forms

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_garbage_output_scores_wrong

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_empty_content

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_trailing_period_does_not_break_match

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_decimal_continuation_still_blocks

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_unclosed_think_block_is_not_correct

1

0

1

tests/unit/benchmark/test_benchmark_metrics.py

TestAnswerMatching.test_closed_think_block_still_works

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheTable.test_every_probe_has_a_known_kind

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheTable.test_the_historic_prompts_are_unchanged_and_keep_the_evidence_kinds

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheTable.test_previews_are_unique_so_they_can_key_a_case

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheTable.test_the_verdict_no_longer_rests_on_two_multiplications

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheTable.test_the_table_is_hashed_into_the_speed_report

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheTable.test_every_integrity_item_asks_for_a_bare_answer

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheEvidence.test_every_campaign_model_passes_every_integrity_item_it_was_asked

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheEvidence.test_every_campaign_report_reads_ok_on_integrity

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheEvidence.test_the_small_instruct_models_miss_only_capability_items

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_capability_misses_alone_are_ok

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_one_wrong_integrity_answer_is_degraded

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_half_the_integrity_answers_wrong_is_broken

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_an_integrity_answer_cut_off_is_inconclusive

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_a_capability_answer_cut_off_is_not

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_garbage_is_broken

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheVerdict.test_no_scored_integrity_answer_is_no_result

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheExitCode.test_capability_misses_exit_zero

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheExitCode.test_an_integrity_miss_exits_one

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheExitCode.test_an_integrity_cut_exits_two

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheExitCode.test_an_unreachable_endpoint_exits_one

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheExitCode.test_main_exits_on_the_integrity_items

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheRecord.test_the_old_fields_still_count_every_item

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheRecord.test_each_kind_is_counted_apart_and_each_item_names_its_kind

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheRecord.test_every_answer_errored_is_no_block

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestTheRecord.test_the_runner_records_kinds

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestOldReports.test_an_old_block_takes_its_kinds_from_the_prompts

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestOldReports.test_a_block_without_items_is_judged_whole_as_before

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestOldReports.test_an_item_the_table_no_longer_has_counts_as_integrity

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestOldReports.test_annotate_gives_an_old_block_what_a_new_one_records

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestOldReports.test_annotate_leaves_a_new_block_alone

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestThePrintout.test_capability_is_its_own_line

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestThePrintout.test_an_integrity_miss_names_the_kernels

1

0

1

tests/unit/benchmark/test_correctness_kinds.py

TestThePrintout.test_no_block_is_no_result

1

0

1

tests/unit/benchmark/test_serving_evidence.py

test_the_registry_pins_this_repos_prompt_file_by_its_raw_bytes

1

0

1

tests/unit/benchmark/test_serving_evidence.py

test_the_pinned_bytes_are_the_crlf_copy

1

0

1

tests/unit/benchmark/test_serving_evidence.py

test_every_p81_report_measured_the_pinned_bytes

2

0

2

tests/unit/benchmark/test_speed_summary.py

TestRatesArePooled.test_decode_is_tokens_after_the_first_over_their_seconds

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestRatesArePooled.test_overall_counts_completion_tokens_only

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestRatesArePooled.test_prefill_is_prompt_tokens_over_their_ttfts

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestRatesArePooled.test_ttft_is_a_plain_mean_of_times

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestErroredRows.test_an_errored_row_is_counted_not_averaged

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestErroredRows.test_an_error_with_an_empty_message_is_still_an_error

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestErroredRows.test_a_run_that_only_errored_has_no_figures

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestCutAndThinkingRows.test_a_cut_row_counts_in_every_rate

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestCutAndThinkingRows.test_a_thinking_only_row_counts_like_any_other

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestZeroAndOneTokenRows.test_a_zero_token_row_slows_overall_and_leaves_decode_alone

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestZeroAndOneTokenRows.test_a_one_token_row_has_no_decode

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestUnstreamedAndEstimatedRows.test_a_non_streamed_run_has_no_ttft_decode_or_prefill

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestUnstreamedAndEstimatedRows.test_chunk_counted_rows_are_counted_and_said

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryPrinterPrintsOneHeadline.test_decode_overall_and_ttft_agree

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryPrinterPrintsOneHeadline.test_the_old_names_are_gone

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryPrinterPrintsOneHeadline.test_the_summary_header_counts_the_errored_request

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryPrinterPrintsOneHeadline.test_an_error_without_a_message_is_an_error_to_every_printer

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestTheTrackedRunReproducesThePage.test_decode_and_ttft

5

0

5

tests/unit/benchmark/test_speed_summary.py

TestTheTrackedRunReproducesThePage.test_the_upgrade_changes

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestTheTrackedRunReproducesThePage.test_overall_is_what_the_lane_generated

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestTheTrackedRunReproducesThePage.test_a_long_reply_run_reads_its_long_replies

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestThePrintedLinesCarryTheSummary.test_the_tracked_run_prints_its_own_figures

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestThePrintedLinesCarryTheSummary.test_the_average_reply_and_the_report_lines_leave_the_error_out

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestABurstRowHasNoRate.test_the_row_says_why

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestABurstRowHasNoRate.test_the_pooled_rate_counts_it_as_before

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestABurstRowHasNoRate.test_the_per_request_range_leaves_it_out

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestABurstRowHasNoRate.test_every_printer_takes_it

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryTrackedSpeedReportPoolsAsBefore.test_every_report_is_read

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryTrackedSpeedReportPoolsAsBefore.test_the_pooled_figures_do_not_move

27

0

27

tests/unit/benchmark/test_speed_summary.py

TestEveryTrackedSpeedReportPoolsAsBefore.test_a_report_the_guarded_writer_wrote_stays_as_it_is

1

0

1

tests/unit/benchmark/test_speed_summary.py

TestEveryTrackedSpeedReportPoolsAsBefore.test_the_t8_run_prints_its_real_rates

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestHardware.test_rows_cover_every_documented_field

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestHardware.test_a_gpu_record_becomes_a_row

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestHardware.test_a_gpu_record_without_memory_still_names_the_card

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestHardware.test_a_missing_record_renders_nothing

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestHardware.test_the_incomplete_list_survives

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectness.test_no_probe_is_not_a_pass

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectness.test_a_full_score_is_correct

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectness.test_half_right_is_degraded_and_less_is_broken

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectness.test_a_failed_item_carries_its_error

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectnessByKind.test_capability_misses_leave_it_correct

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectnessByKind.test_an_integrity_miss_still_degrades_it

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectnessByKind.test_each_row_names_its_kind

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestCorrectnessByKind.test_a_block_without_kinds_is_judged_whole_as_before

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestWilson.test_perfect_score_still_has_a_lower_bound

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestWilson.test_no_trials_is_the_full_range

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestWilson.test_more_trials_narrow_the_interval

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestScoredRows.test_rows_carry_interval_and_sort_by_score

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestScoredRows.test_scored_table_existence

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestComparisonAndCharts.test_average_only_counts_results_carrying_the_metric

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestComparisonAndCharts.test_a_config_without_the_metric_is_dropped_from_the_chart

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestComparisonAndCharts.test_ctx_and_tok_are_read_out_of_the_label

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestComparisonAndCharts.test_gpu_utilization_averages_only_over_prompts_that_have_it

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestSpeedComesFromTheManifest.test_the_table_and_the_charts_read_the_block_not_the_rows

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestSpeedComesFromTheManifest.test_a_manifest_older_than_the_block_shows_dashes_not_a_second_average

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestDetail.test_extra_params_come_first_and_managed_keys_are_hidden

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestDetail.test_per_prompt_rows_degrade_missing_metrics_to_dash

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestDetail.test_per_prompt_rows_carry_gpu_utilization

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestDetail.test_errors_are_listed_separately

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestSummary.test_requests_and_errors_are_counted_apart

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestAnswersAreNotTimeToTheCap.test_an_older_reports_unclosed_think_reads_as_all_thinking

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestAnswersAreNotTimeToTheCap.test_a_cut_row_has_no_time_to_an_answer

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestAnswersAreNotTimeToTheCap.test_an_older_row_without_the_flag_keeps_its_latency

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestAThinkingShareNobodyCanRead.test_the_mean_says_how_many_it_could_not_count

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestAThinkingShareNobodyCanRead.test_a_run_of_unknowns_is_not_a_dash

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestAThinkingShareNobodyCanRead.test_the_per_prompt_cell_tells_unknown_from_unmeasured

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestTheViewerReadsAShareAsTheRunnerDoes.test_every_arm_reads_alike

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestTheViewerReadsAShareAsTheRunnerDoes.test_every_tracked_row_reads_alike

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestPerPromptLabFields.test_the_new_fields_render

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestPerPromptLabFields.test_other_load_is_derived_for_an_older_row_and_says_so

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestPerPromptLabFields.test_a_snapshot_cpu_reading_derives_nothing

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestPerPromptLabFields.test_an_old_row_renders_dashes_not_zeros

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestScoredCountsAreCounts.test_effective_k_is_used_rather_than_a_rounded_ratio

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestManifestLocation.test_a_relative_path_is_read_from_the_repository_root

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestManifestLocation.test_a_path_that_exists_from_the_working_directory_wins

1

0

1

tests/unit/frontend/test_benchmark_data.py

TestManifestLocation.test_an_absolute_path_is_taken_as_given

1

0

1

tests/unit/frontend/test_lab_data.py

TestAnswers.test_answered_counts_and_first_answer_covers_answered_rows_only

1

0

1

tests/unit/frontend/test_lab_data.py

TestAnswers.test_all_answered_is_not_cut

1

0

1

tests/unit/frontend/test_lab_data.py

TestAnswers.test_a_report_older_than_the_flag_shows_a_dash

1

0

1

tests/unit/frontend/test_lab_data.py

TestAnswers.test_thinking_share_repairs_an_unclosed_think

1

0

1

tests/unit/frontend/test_lab_data.py

TestAnswers.test_thinking_share_counts_the_cut_replies_it_cannot_read

1

0

1

tests/unit/frontend/test_lab_data.py

TestLoad.test_lane_mean_and_other_mean_with_max

1

0

1

tests/unit/frontend/test_lab_data.py

TestLoad.test_an_older_report_derives_other_load_and_marks_it

1

0

1

tests/unit/frontend/test_lab_data.py

TestLoad.test_without_the_thread_count_nothing_is_derived

1

0

1

tests/unit/frontend/test_lab_data.py

TestLoad.test_zero_other_load_is_a_reading_not_a_gap

1

0

1

tests/unit/frontend/test_lab_data.py

TestEnergy.test_joules_per_token_is_a_ratio_of_sums

1

0

1

tests/unit/frontend/test_lab_data.py

TestEnergy.test_net_only_when_every_metered_row_has_it

1

0

1

tests/unit/frontend/test_lab_data.py

TestEnergy.test_power_falls_back_to_latency_for_an_older_row

1

0

1

tests/unit/frontend/test_lab_data.py

TestEnergy.test_an_unmetered_run_renders_dashes

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_a_drifted_baseline_says_read_gross

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_a_steady_baseline_is_reliable

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_one_baseline_is_unknown_not_reliable

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_a_report_older_than_the_flag_is_unknown

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_no_meter_names_the_reason

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_no_block_at_all

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_the_flag_reaches_the_lab_row

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_a_metered_run_without_a_baseline_is_gross_only

1

0

1

tests/unit/frontend/test_lab_data.py

TestNetReliability.test_an_older_report_whose_rows_were_netted_stays_unknown

1

0

1

tests/unit/frontend/test_lab_data.py

TestLabRowSelection.test_runs_without_any_lab_field_get_no_row

1

0

1

tests/unit/frontend/test_lab_data.py

TestLabRowSelection.test_errored_rows_are_left_out

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_a_geniex_build_is_named_in_full

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_other_servers_and_missing_runtimes

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_serve_flags_drop_the_subcommand_and_the_host

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_an_unseen_lane_process_has_no_flags

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_runtime_rows_carry_lane_endpoint_and_how_it_was_seen

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_an_installed_binary_is_not_the_lane_process

1

0

1

tests/unit/frontend/test_lab_data.py

TestRuntime.test_an_envelope_takes_lane_and_model_from_its_reports

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_columns_group_by_lane_and_rows_follow_the_checks

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_the_header_names_lane_and_version

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_a_check_added_later_sits_where_its_run_put_it

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_moved_marks_an_answer_that_changed_within_one_lane

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_a_lane_reads_oldest_first_whatever_the_files_are_called

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_a_check_one_run_never_asked_moves_nothing

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_errors_keep_their_state_and_evidence_carries_the_seconds

1

0

1

tests/unit/frontend/test_lab_data.py

TestContractTable.test_other_kinds_are_not_contract_columns

1

0

1

tests/unit/frontend/test_page_compiles.py

test_the_page_builds_and_compiles

1

0

1

tests/unit/frontend/test_viewer_job.py

test_the_viewer_tests_collect_without_the_package

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackCreation.test_first_update_creates_one_track_per_centroid

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackCreation.test_empty_update_returns_existing_tracks_untouched

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackCreation.test_far_detection_spawns_new_track

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackMatching.test_nearby_detection_extends_same_track

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackMatching.test_greedy_matching_prefers_closest_pair

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackMatching.test_trail_respects_max_points

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackExpiry.test_stale_track_expires

1

0

1

tests/unit/test_centroid_tracker.py

TestTrackExpiry.test_fresh_track_survives_expiry_sweep

1

0

1

tests/unit/test_dummy.py

test_generate_synthetic_data

1

0

1

tests/unit/test_dummy.py

test_normalize_features

1

0

1

tests/unit/test_dummy.py

test_apply_joke_labeling

1

0

1

tests/unit/test_fetch_models.py

TestCacheDir.test_env_wins

1

0

1

tests/unit/test_fetch_models.py

TestCacheDir.test_platform_default

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_downloads_into_dest_and_cache

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_sha_mismatch_fails_and_writes_nothing

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_cache_hit_skips_the_download

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_corrupt_cache_entry_is_downloaded_again

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_correct_target_is_kept

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_wrong_target_is_replaced

1

0

1

tests/unit/test_fetch_models.py

TestFetch.test_unreadable_url_fails_after_retries

1

0

1

tests/unit/test_fetch_models.py

TestLockAndArguments.test_dest_defaults_to_the_locks_directory

1

0

1

tests/unit/test_fetch_models.py

TestLockAndArguments.test_missing_lock_is_a_usage_error

1

0

1

tests/unit/test_fetch_models.py

TestLockAndArguments.test_malformed_rows_are_refused

2

0

2

tests/unit/test_fetch_models.py

TestLockAndArguments.test_repo_lock_carries_the_default_model

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_reads_metrics_and_orders_largest_vram_first

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_second_index_is_the_smaller_card

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_non_amd_vendors_are_ignored

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_power1_input_is_the_fallback

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_missing_fields_degrade_to_zero

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_out_of_range_index_is_unavailable

1

0

1

tests/unit/test_gpu_amd.py

TestSysfsBackend.test_no_amd_card_at_all

1

0

1

tests/unit/test_gpu_amd.py

TestAdlBackend.test_selects_the_adapter_with_the_most_vram

1

0

1

tests/unit/test_gpu_amd.py

TestAdlBackend.test_maps_pmlog_sensors_to_a_snapshot

1

0

1

tests/unit/test_gpu_amd.py

TestAdlBackend.test_hotspot_temp_and_board_power_are_the_fallbacks

1

0

1

tests/unit/test_gpu_amd.py

TestAdlBackend.test_empty_adapter_list_closes_the_context

1

0

1

tests/unit/test_gpu_amd.py

TestAdlBackend.test_out_of_range_index_closes_the_context

1

0

1

tests/unit/test_gpu_amd.py

TestAdlBackend.test_shutdown_closes_the_context

1

0

1

tests/unit/test_gpu_amd.py

TestGpuProbeFacade.test_falls_back_to_amd_when_nvml_is_absent

1

0

1

tests/unit/test_gpu_amd.py

TestGpuProbeFacade.test_vendor_nvidia_does_not_fall_back_to_amd

1

0

1

tests/unit/test_gpu_amd.py

TestGpuProbeFacade.test_no_backend_at_all

1

0

1

tests/unit/test_gstreamer_env.py

test_gst_variables_reach_the_subprocess

1

0

1

tests/unit/test_gstreamer_env.py

test_without_gst_variables_only_the_allowlist

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_initialization_empty_metrics

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_initialization_with_metrics

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_initialization_with_gpu_metrics

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_prepare_time_data

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_plot_cpu_memory

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_plot_gpu_without_gpu_data

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_plot_gpu_with_gpu_data

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_plot_all_without_gpu

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_plot_all_with_gpu

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_save_figure

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_save_figure_without_plot

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_show

1

0

1

tests/unit/test_metrics_plotter.py

TestMetricsPlotter.test_quick_plot

1

0

1

tests/unit/test_metrics_plotter.py

test_importing_the_package_loads_no_pyplot

1

0

1

tests/unit/test_paths.py

TestBundleDataDir.test_env_wins

1

0

1

tests/unit/test_paths.py

TestBundleDataDir.test_bundle_layout_beside_the_interpreter

1

0

1

tests/unit/test_paths.py

TestBundleDataDir.test_plain_venv_is_not_a_bundle

1

0

1

tests/unit/test_paths.py

TestModelPath.test_env_first_then_bundle_then_checkout

1

0

1

tests/unit/test_paths.py

TestModelPath.test_first_existing_candidate_wins

1

0

1

tests/unit/test_paths.py

TestModelPath.test_nothing_found_names_the_first_candidate

1

0

1

tests/unit/test_paths.py

TestModelPath.test_missing_model_names_the_fetch_command

1

0

1

tests/unit/test_paths.py

TestLogDir.test_checkout_keeps_relative_logs

1

0

1

tests/unit/test_paths.py

TestLogDir.test_env_wins

1

0

1

tests/unit/test_paths.py

TestLogDir.test_bundle_logs_to_the_user_state_dir

1

0

1

tests/unit/test_performance_tracker.py

TestCameraFps.test_steady_ticks_yield_exact_fps

1

0

1

tests/unit/test_performance_tracker.py

TestCameraFps.test_no_ticks_reports_zero_fps

1

0

1

tests/unit/test_performance_tracker.py

TestCameraFps.test_rolling_window_caps_history

1

0

1

tests/unit/test_performance_tracker.py

TestInferenceMetrics.test_average_and_capacity

1

0

1

tests/unit/test_performance_tracker.py

TestInferenceMetrics.test_frame_budget_percent

1

0

1

tests/unit/test_performance_tracker.py

TestThroughput.test_actual_throughput_counts_all_frames

1

0

1

tests/unit/test_smoke_checks.py

TestCheckContract.test_run_all_returns_results_for_every_check

1

0

1

tests/unit/test_smoke_checks.py

TestLaneDependentWheels.test_missing_onnxruntime_genai_is_optional

1

0

1

tests/unit/test_smoke_checks.py

TestLaneDependentWheels.test_missing_tvm_is_optional

1

0

1

tests/unit/test_smoke_checks.py

TestLaneDependentWheels.test_missing_pyav_is_optional

1

0

1

tests/unit/test_smoke_checks.py

TestLaneDependentWheels.test_missing_iree_is_optional

1

0

1

tests/unit/test_smoke_checks.py

TestLaneDependentWheels.test_missing_litert_is_optional

1

0

1

tests/unit/test_system_monitor.py

TestSystemMetrics.test_create_metrics_basic

1

0

1

tests/unit/test_system_monitor.py

TestSystemMetrics.test_create_metrics_with_gpu

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_monitor_initialization

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_start_stop_monitoring

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_collect_metrics

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_multiple_records

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_get_metrics

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_print_summary_no_metrics

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_print_summary_with_metrics

1

0

1

tests/unit/test_system_monitor.py

TestSystemMonitor.test_no_gpu_available

1

0

1

tests/unit/test_thread_safety.py

test_every_gateway_reply_is_counted

1

0

1

tests/unit/test_thread_safety.py

test_concurrent_first_callers_share_one_gpu_probe

1

0

1

tests/unit/test_thread_safety.py

test_a_queued_frame_is_not_overwritten_by_the_next_read

1

0

1

tests/unit/test_thread_safety.py

test_release_while_the_reader_runs_raises_nothing_on_the_reader

1

0

1

tests/unit/test_thread_safety.py

test_offering_frames_from_many_threads_never_blocks_or_overflows

1

0

1

tests/unit/test_thread_safety.py

test_streams_sharing_one_capture_never_raise_on_an_empty_queue

1

0

1

tests/unit/test_thread_safety.py

test_the_log_buffer_reads_while_sinks_append

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_valid_config

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_default_config

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_device_index

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_width_too_small

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_width_too_large

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_height_too_small

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_height_too_large

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_fps_too_low

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_invalid_fps_too_high

1

0

1

tests/unit/test_types.py

TestCameraConfig.test_boundary_values

1

0

1

tests/unit/test_yolo_cli.py

test_model_defaults_to_the_resolved_path

1

0

1

tests/unit/test_yolo_cli.py

test_explicit_model_is_kept

1

0

1

tests/unit/test_yolo_cli.py

test_providers_follow_the_build

3

0

3

tests/unit/test_yolo_postprocess.py

TestMathHelpers.test_softmax_is_a_probability_distribution

1

0

1

tests/unit/test_yolo_postprocess.py

TestMathHelpers.test_sigmoid_midpoint_and_monotonicity

1

0

1

tests/unit/test_yolo_postprocess.py

TestMathHelpers.test_xywh_to_xyxy_known_box

1

0

1

tests/unit/test_yolo_postprocess.py

TestMathHelpers.test_squeeze_to_2d_drops_batch_dim

1

0

1

tests/unit/test_yolo_postprocess.py

TestFormatHeuristics.test_xyxy_boxes_are_not_flagged_as_xywh

1

0

1

tests/unit/test_yolo_postprocess.py

TestFormatHeuristics.test_center_boxes_are_flagged_as_xywh

1

0

1

tests/unit/test_yolo_postprocess.py

TestFormatHeuristics.test_empty_boxes_are_not_xywh

1

0

1

tests/unit/test_yolo_postprocess.py

TestFormatHeuristics.test_prepare_boxes_scales_normalized_coordinates

1

0

1

tests/unit/test_yolo_postprocess.py

TestClassificationDecode.test_logit_vector_decodes_to_class

1

0

1

tests/unit/test_yolo_postprocess.py

TestClassificationDecode.test_detection_matrix_is_not_classification

1

0

1

tests/unit/test_yolo_postprocess.py

TestUnscaleAndCollect.test_threshold_filters_low_scores

1

0

1

tests/unit/test_yolo_postprocess.py

TestUnscaleAndCollect.test_padding_and_scale_are_undone

1

0

1

tests/unit/test_yolo_postprocess.py

TestUnscaleAndCollect.test_per_axis_scale_is_undone

1

0

1

tests/unit/test_yolo_postprocess.py

TestPostprocessEntry.test_empty_outputs_return_nothing

1

0

1

tests/unit/test_yolo_postprocess.py

TestPostprocessEntry.test_end_to_end_output_maps_back_per_axis

1

0

1

tests/unit/test_yolo_postprocess.py

TestPostprocessEntry.test_classification_output_takes_priority

1

0

1

tests/unit/test_yolo_preprocess.py

TestWantsStretch.test_stretch_metadata

1

0

1

tests/unit/test_yolo_preprocess.py

TestWantsStretch.test_any_other_model_keeps_the_letterbox

2

0

2

tests/unit/test_yolo_preprocess.py

TestStretch.test_fills_the_input_without_padding

1

0

1

tests/unit/test_yolo_preprocess.py

TestStretch.test_letterbox_is_unchanged

1

0

1

tests/unit/test_yolo_preprocess.py

TestStretch.test_boxes_map_back_to_the_frame

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixtures.test_every_task_has_a_reference_solution

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixtures.test_reference_only_touches_files_in_the_fixture

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixtures.test_task_names_are_unique

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixtures.test_prompt_never_contains_the_solution

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixtures.test_every_cheat_names_a_task_and_a_refusal

1

0

1

benchmarks/tests/test_bench_agent.py

TestWorkspace.test_workspace_gets_the_fixture_files_and_a_git_repo

1

0

1

benchmarks/tests/test_bench_agent.py

TestWorkspace.test_workspaces_are_independent

1

0

1

benchmarks/tests/test_bench_agent.py

TestWorkspace.test_bytecode_is_not_part_of_the_diff

1

0

1

benchmarks/tests/test_bench_agent.py

TestVerification.test_unsolved_fixture_fails

6

0

6

benchmarks/tests/test_bench_agent.py

TestVerification.test_reference_solution_passes

6

0

6

benchmarks/tests/test_bench_agent.py

TestVerification.test_missing_command_is_a_failure_not_a_crash

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_deleted_test_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_skipped_test_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_inverted_assertion_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_deleted_test_file_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_added_conftest_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_an_added_pytest_config_is_refused

3

0

3

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_an_edited_test_is_refused_even_when_the_fix_is_real

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_unrelated_new_files_are_allowed

1

0

1

benchmarks/tests/test_bench_agent.py

TestFixTaskProtectsItsTests.test_a_broken_git_is_a_refusal_not_a_pass

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_correct_clamp_with_no_tests_written_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_vacuous_test_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_missing_error_case_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_missing_range_tests_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_only_one_end_of_the_range_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_full_tests_pass

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_clamp_task_rejects_a_vacuous_test

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_reference_tests_kill_every_mutant_and_fixture_tests_kill_none

1

0

1

benchmarks/tests/test_bench_agent.py

TestClampTaskRequiresRealTests.test_no_tests_at_all_is_not_counted_as_a_kill

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_comment_mentioning_the_old_name_passes

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_docstring_mentioning_the_old_name_passes

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_a_bare_docstring_of_exactly_the_old_name_passes

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_identifier_containing_the_old_name_passes

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_rename_task_rejects_an_alias

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_wrapper_def_is_rejected

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_globals_lookup_is_rejected

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_import_alias_is_rejected

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_attribute_use_is_rejected

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_unparseable_file_falls_back_to_text

1

0

1

benchmarks/tests/test_bench_agent.py

TestRenameTaskDecidesOnTheTree.test_detail_is_never_empty

1

0

1

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_context_overflow_is_not_a_capability_result

1

0

1

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_every_marker_is_recognised

4

0

4

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_overflow_after_tool_calls_is_context_growth_not_blocked

1

0

1

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_a_step_alone_means_the_model_was_reached

1

0

1

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_a_bare_context_or_too_long_is_a_plain_error

4

0

4

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_plain_errors_keep_their_message

1

0

1

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_no_errors_is_empty

1

0

1

benchmarks/tests/test_bench_agent.py

TestErrorClassification.test_non_error_events_are_never_classified

1

0

1

benchmarks/tests/test_bench_agent.py

TestEventSummary.test_unrecognised_events_are_counted_not_dropped

1

0

1

benchmarks/tests/test_bench_agent.py

TestEventSummary.test_counts_add_up

1

0

1

benchmarks/tests/test_bench_agent.py

TestCli.test_self_test_passes

1

0

1

benchmarks/tests/test_bench_agent.py

TestCli.test_self_test_reports_a_cheat_that_slips_through

1

0

1

benchmarks/tests/test_bench_agent.py

TestCli.test_list_needs_no_server

1

0

1

benchmarks/tests/test_bench_agent.py

TestCli.test_unknown_task_exits_nonzero_and_writes_no_report

1

0

1

benchmarks/tests/test_bench_agent.py

TestTimeoutKeepsEvidence.test_partial_output_survives_a_timeout_and_the_tree_dies

1

0

1

benchmarks/tests/test_bench_agent.py

TestTimeoutKeepsEvidence.test_no_output_at_all_is_still_safe

1

0

1

benchmarks/tests/test_bench_agent.py

TestTimeoutKeepsEvidence.test_a_finished_run_is_not_a_timeout

1

0

1

benchmarks/tests/test_bench_agent.py

TestTimeoutKeepsEvidence.test_env_is_passed_to_the_agent

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_jsonc_comments_and_trailing_commas_are_tolerated

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_a_url_inside_a_string_is_not_a_comment

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_base_url_resolves_from_the_models_provider

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_unresolvable_is_none_not_a_guess

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_config_is_hashed_even_when_it_does_not_parse

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_missing_config_is_null_with_a_note

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_config_path_honours_opencode_config_then_xdg

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_env_points_sessions_at_the_scratch_dir_and_carries_auth

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_env_without_a_config_sets_no_opencode_config

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_version_is_null_when_the_binary_is_missing

1

0

1

benchmarks/tests/test_bench_agent.py

TestOpencodeProvenance.test_version_is_read_from_the_binary

1

0

1

benchmarks/tests/test_bench_agent.py

TestScoreExcludesBlockedRuns.test_stats_layer_renders_nothing_measured_as_na

1

0

1

benchmarks/tests/test_bench_agent.py

TestScoreExcludesBlockedRuns.test_blocked_rows_are_errored_and_out_of_the_denominator_and_wall

1

0

1

benchmarks/tests/test_bench_agent.py

TestScoreExcludesBlockedRuns.test_context_growth_is_a_real_fail_in_the_denominator

1

0

1

benchmarks/tests/test_bench_agent.py

TestScoreExcludesBlockedRuns.test_score_is_printed_with_its_interval

1

0

1

benchmarks/tests/test_bench_agent.py

TestReportProvenance.test_provenance_carries_opencode_and_lane

1

0

1

benchmarks/tests/test_bench_agent.py

TestReportProvenance.test_unresolvable_lane_is_null_and_listed_incomplete

1

0

1

benchmarks/tests/test_bench_agent.py

TestReportProvenance.test_broken_config_is_listed_incomplete

1

0

1

benchmarks/tests/test_bench_agent.py

TestReportProvenance.test_diff_sha_is_always_stored_and_the_diff_only_on_request

1

0

1

benchmarks/tests/test_bench_agent.py

TestReportProvenance.test_diff_is_bounded

1

0

1

benchmarks/tests/test_bench_agent.py

TestReportProvenance.test_scratch_data_home_is_removed_after_the_run

1

0

1

benchmarks/tests/test_bench_agent.py

TestWriteReportExtra.test_extra_fields_land_in_the_provenance_block

1

0

1

benchmarks/tests/test_bench_agent.py

TestWriteReportExtra.test_incomplete_extends_rather_than_replaces

1

0

1

benchmarks/tests/test_bench_agent.py

TestWriteReportExtra.test_no_extra_changes_nothing

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_the_fixture_set_is_not_python_only

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_each_new_fixture_declares_the_tools_it_needs

2

0

2

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_missing_tools_lists_only_what_is_absent

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_alternatives_are_satisfied_by_either

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_a_missing_alternative_group_is_named_in_full

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_a_task_with_no_requires_needs_nothing

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_the_bash_fixture_starts_red

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_the_bash_reference_turns_it_green

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_quoting_only_half_the_bug_still_fails

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_editing_the_check_script_is_refused

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_the_c_test_of_the_cmake_fixture_is_protected

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_the_cmake_verify_refuses_a_run_with_no_tests

1

0

1

benchmarks/tests/test_bench_agent.py

TestOtherLanguageFixtures.test_the_cmake_fixture_is_missing_exactly_the_link_line

1

0

1

benchmarks/tests/test_bench_agent.py

TestProtectionIsNotVacuous.test_protect_tests_without_a_test_file_fails_loudly

1

0

1

benchmarks/tests/test_bench_agent.py

TestProtectionIsNotVacuous.test_a_task_with_a_test_file_is_checked_normally

1

0

1

benchmarks/tests/test_bench_agent.py

TestSkipsAreVisible.test_self_test_skips_rather_than_failing_when_cmake_is_absent

1

0

1

benchmarks/tests/test_bench_agent.py

TestSkipsAreVisible.test_a_skipped_fixture_is_never_reported_as_validated

1

0

1

benchmarks/tests/test_bench_agent.py

TestSkipsAreVisible.test_a_broken_fixture_still_fails_while_another_is_skipped

1

0

1

benchmarks/tests/test_bench_agent.py

TestSkipsAreVisible.test_selecting_only_an_unbuildable_task_exits_nonzero

1

0

1

benchmarks/tests/test_bench_agent.py

TestSkipsAreVisible.test_the_report_records_which_fixtures_were_skipped

1

0

1

benchmarks/tests/test_bench_agent.py

TestSkipsAreVisible.test_nothing_is_skipped_when_every_tool_is_present

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_it_is_a_medium_repo

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_every_python_file_parses

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_no_carriage_returns_or_trailing_blanks

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_the_fix_is_not_where_the_failures_are

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_the_reference_changes_only_the_parser

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_the_prompt_does_not_name_the_file

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_a_stale_anchor_fails_loudly

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_its_self_test_rows_fit_the_column

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_it_is_wired_into_the_harness

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestShape.test_the_report_fingerprints_both_fixture_modules

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestRedThenGreen.test_exactly_the_planted_tests_are_red

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestRedThenGreen.test_the_reference_makes_the_whole_suite_green

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestRedThenGreen.test_the_unseen_inputs_fail_unfixed_and_pass_fixed

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestRedThenGreen.test_verify_is_red_before_and_green_after

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestRedThenGreen.test_the_self_test_proves_it_and_refuses_its_cheats

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_each_cheat_would_turn_the_suite_green_on_its_own

3

0

3

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_each_cheat_is_refused_for_its_reason

3

0

3

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_a_pytest_ini_at_the_root_is_refused

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_deselecting_through_pyproject_leaves_the_red_tests_red

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_skipping_the_red_tests_by_name_is_refused

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_a_new_test_module_patching_the_parser_is_refused

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_a_fix_in_both_callers_is_accepted

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_a_crashing_program_is_refused_with_its_error

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestCheatsAreRefused.test_output_that_does_not_decode_fails_the_trial_not_the_run

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestNestedFixtures.test_a_nested_fixture_is_written_and_committed_clean

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestNestedFixtures.test_apply_files_clears_bytecode_in_every_package

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestNestedFixtures.test_an_override_file_counts_in_any_directory_above_the_tests

1

0

1

benchmarks/tests/test_bench_agent_medium.py

TestNestedFixtures.test_a_flat_fixture_is_judged_as_before

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_the_agent_uses_the_shared_helper

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_a_task_that_always_passes_is_one_at_every_k

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_a_task_that_never_passes_is_zero_at_every_k

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_two_of_three

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_the_mean_runs_over_tasks_not_over_trials

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_a_task_with_fewer_than_k_trials_is_left_out_at_that_k

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_no_task_reaching_k_is_none_not_zero

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_k_below_one_is_refused

2

0

2

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_it_is_the_share_of_k_subsets_of_trials_that_all_passed

20

0

20

benchmarks/tests/test_bench_agent_repeats.py

TestPassHatK.test_it_never_rises_with_k

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestSummariseTrials.test_per_task_counts_and_pass_hat_k

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestSummariseTrials.test_the_interval_is_stored_and_is_the_printed_one

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestSummariseTrials.test_nothing_measured_stores_no_interval

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestSummariseTrials.test_a_blocked_trial_is_not_a_failed_attempt

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_every_trial_is_a_row_with_its_attempt

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_the_report_carries_per_task_counts_pass_hat_k_and_the_interval

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_each_trial_gets_a_fresh_workspace_and_opencode_home

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_trials_run_round_robin_over_the_tasks

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_a_blocked_trial_leaves_the_attempts_not_the_task

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_the_default_is_one_trial_and_prints_no_pass_hat

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_repeats_print_the_per_task_table_and_pass_hat

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_a_repeat_count_below_one_is_refused

3

0

3

benchmarks/tests/test_bench_agent_repeats.py

TestRepeatsPlumbing.test_bench_compare_reads_repeats_as_attempts_of_one_case

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestOpencodeState.test_state_lands_in_the_scratch_home

1

0

1

benchmarks/tests/test_bench_agent_repeats.py

TestOpencodeState.test_config_and_cache_are_left_alone

1

0

1

benchmarks/tests/test_bench_chat.py

TestEveryCase.test_every_hand_written_case_has_a_good_and_a_bad_reply

1

0

1

benchmarks/tests/test_bench_chat.py

TestEveryCase.test_the_good_reply_passes_and_the_bad_one_fails

24

0

24

benchmarks/tests/test_bench_chat.py

TestEveryCase.test_a_document_answer_must_name_the_value_and_not_the_decoy

9

0

9

benchmarks/tests/test_bench_chat.py

TestEveryCase.test_about_thirty_cases_in_the_four_families

1

0

1

benchmarks/tests/test_bench_chat.py

TestEveryCase.test_a_json_case_forbids_a_fence_and_says_so

1

0

1

benchmarks/tests/test_bench_chat.py

TestEveryCase.test_multi_turn_cases_end_on_the_user_and_refer_back

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_words_count_hyphenated_and_apostrophised_words_once

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_sentences_do_not_split_on_decimals_but_do_after_quotes

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_an_abbreviation_does_not_end_a_sentence

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_a_sign_off_alone_names_no_planet

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_paragraphs_fall_back_to_lines_when_no_blank_line_separates_them

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_german_votes

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_markdown_is_named_by_kind

9

0

9

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_plain_prose_with_a_hyphen_and_an_asterisk_is_not_markdown

1

0

1

benchmarks/tests/test_bench_chat.py

TestTextMeasures.test_mentions_matches_whole_words_and_whole_numbers

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_integer_is_not_a_bool_and_three_point_zero_is_an_integer

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_a_type_list_and_null

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_enum_and_const_do_not_confuse_true_with_one

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_object_errors_name_the_path

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_limits

6

0

6

benchmarks/tests/test_bench_chat.py

TestValidator.test_an_unimplemented_keyword_raises_rather_than_passing

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_every_case_schema_uses_only_implemented_keywords

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_parse_json_reply

1

0

1

benchmarks/tests/test_bench_chat.py

TestValidator.test_capitals_must_be_distinct

1

0

1

benchmarks/tests/test_bench_chat.py

TestDocuments.test_the_same_document_every_run

1

0

1

benchmarks/tests/test_bench_chat.py

TestDocuments.test_sized_under_the_nominal_and_near_it

3

0

3

benchmarks/tests/test_bench_chat.py

TestDocuments.test_the_npu_sized_document_leaves_room_in_4096

1

0

1

benchmarks/tests/test_bench_chat.py

TestDocuments.test_each_value_and_decoy_appears_once

3

0

3

benchmarks/tests/test_bench_chat.py

TestDocuments.test_filler_numbers_never_collide_with_an_answer

1

0

1

benchmarks/tests/test_bench_chat.py

TestDocuments.test_approx_tokens_counts_digits_singly

1

0

1

benchmarks/tests/test_bench_chat.py

TestGradeReply.test_thinking_is_stripped_before_grading

1

0

1

benchmarks/tests/test_bench_chat.py

TestGradeReply.test_an_answer_only_inside_an_unclosed_think_is_no_answer

1

0

1

benchmarks/tests/test_bench_chat.py

TestGradeReply.test_a_separate_reasoning_field_is_not_the_answer

1

0

1

benchmarks/tests/test_bench_chat.py

TestGradeReply.test_a_reply_cut_at_the_budget_is_not_graded

1

0

1

benchmarks/tests/test_bench_chat.py

TestGradeReply.test_the_hash_covers_thinking_too

1

0

1

benchmarks/tests/test_bench_chat.py

TestRunCase.test_a_pass_records_the_usage_and_the_document_size

1

0

1

benchmarks/tests/test_bench_chat.py

TestRunCase.test_a_multi_turn_case_sends_its_whole_history

1

0

1

benchmarks/tests/test_bench_chat.py

TestRunCase.test_a_context_refusal_is_an_overflow_not_a_failure

1

0

1

benchmarks/tests/test_bench_chat.py

TestRunCase.test_anything_else_is_a_transport_error

4

0

4

benchmarks/tests/test_bench_chat.py

TestRunCase.test_a_reply_that_is_not_an_object_is_an_error

1

0

1

benchmarks/tests/test_bench_chat.py

TestEvaluate.test_repeats_are_spaced_and_identical_replies_are_one_observation

1

0

1

benchmarks/tests/test_bench_chat.py

TestEvaluate.test_differing_replies_with_the_same_verdicts_still_count_per_case

1

0

1

benchmarks/tests/test_bench_chat.py

TestEvaluate.test_mixed_verdicts_count_every_attempt

1

0

1

benchmarks/tests/test_bench_chat.py

TestEvaluate.test_rows_nobody_graded_leave_the_denominator_and_are_counted

1

0

1

benchmarks/tests/test_bench_chat.py

TestEvaluate.test_a_failed_warmup_does_not_stop_the_run

1

0

1

benchmarks/tests/test_bench_chat.py

TestEvaluate.test_the_report_reads_back_through_bench_compare

1

0

1

benchmarks/tests/test_bench_chat.py

TestMain.test_a_category_run_writes_the_envelope

1

0

1

benchmarks/tests/test_bench_chat.py

TestMain.test_the_fingerprint_is_this_file_and_the_probe

1

0

1

benchmarks/tests/test_bench_chat.py

TestMain.test_the_run_start_record_reaches_the_report

1

0

1

benchmarks/tests/test_bench_chat.py

TestMain.test_no_output_writes_nothing

1

0

1

benchmarks/tests/test_bench_chat.py

TestMain.test_suspect_cases_leave_the_categories_in_this_tools_shape

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_fenced_python_block

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_unfenced_code_is_still_graded

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_thinking_block_is_dropped

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_unclosed_thinking_block_yields_nothing

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_prefers_the_block_defining_the_required_function

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_falls_back_to_a_block_containing_any_def

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_longest_wins_only_when_no_block_defines_anything

1

0

1

benchmarks/tests/test_bench_coding.py

TestExtraction.test_no_code_yields_empty

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_correct_implementation_passes

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_plausible_but_wrong_fails

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_infinite_loop_is_caught_not_hung

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_empty_reply_fails_with_a_clear_reason

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_syntax_error_fails

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_wrong_function_name_fails

1

0

1

benchmarks/tests/test_bench_coding.py

TestGrading.test_balanced_task_rejects_a_naive_counter

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationDetection.test_hitting_the_generation_cap_counts_as_truncated

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationDetection.test_a_server_reported_length_finish_is_a_cut_without_any_cap

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationDetection.test_short_reply_is_not_truncated

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationDetection.test_unclosed_fence_with_broken_code_is_truncated

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationDetection.test_valid_code_in_a_closed_fence_is_never_truncated

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationDetection.test_a_plain_typo_in_a_closed_fence_is_wrong_not_truncated

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationTails.test_a_closing_fence_is_not_an_unclosed_opener

6

0

6

benchmarks/tests/test_bench_coding.py

TestTruncationTails.test_a_genuinely_unclosed_final_fence_is_still_a_cut

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationTails.test_the_server_reported_length_still_wins_over_a_closed_fence

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationTails.test_the_delta_count_cap_still_wins_over_a_closed_fence

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationTails.test_a_cut_landing_on_a_compiling_prefix_is_a_cut_not_a_failure

1

0

1

benchmarks/tests/test_bench_coding.py

TestTruncationOutsidePython.test_a_stop_with_an_unclosed_fence_is_graded_not_cut

4

0

4

benchmarks/tests/test_bench_coding.py

TestTruncationOutsidePython.test_a_stop_with_a_closed_fence_is_graded

4

0

4

benchmarks/tests/test_bench_coding.py

TestTruncationOutsidePython.test_a_length_finish_is_a_cut

8

0

8

benchmarks/tests/test_bench_coding.py

TestTruncationOutsidePython.test_an_unclosed_fence_at_the_token_cap_is_a_cut

8

0

8

benchmarks/tests/test_bench_coding.py

TestTruncationOutsidePython.test_an_unclosed_fence_with_no_finish_reason_is_a_cut

4

0

4

benchmarks/tests/test_bench_coding.py

TestTruncationOutsidePython.test_python_keeps_its_syntax_probe

1

0

1

benchmarks/tests/test_bench_coding.py

TestNonPythonRowsReadFailOrCut.test_complete_but_wrong_with_a_stop_is_fail

4

0

4

benchmarks/tests/test_bench_coding.py

TestNonPythonRowsReadFailOrCut.test_complete_but_wrong_with_a_length_finish_is_cut

4

0

4

benchmarks/tests/test_bench_coding.py

TestNonPythonRowsReadFailOrCut.test_an_unclosed_fence_at_the_token_cap_is_cut

8

0

8

benchmarks/tests/test_bench_coding.py

TestMultiFenceAndIndentedExtraction.test_a_helper_in_an_earlier_fence_is_kept

1

0

1

benchmarks/tests/test_bench_coding.py

TestMultiFenceAndIndentedExtraction.test_an_import_in_an_earlier_fence_is_kept

1

0

1

benchmarks/tests/test_bench_coding.py

TestMultiFenceAndIndentedExtraction.test_a_demo_quoting_the_signature_in_its_docstring_does_not_win

1

0

1

benchmarks/tests/test_bench_coding.py

TestMultiFenceAndIndentedExtraction.test_an_indented_fence_with_two_statements_compiles

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestFences.test_language_variants_are_all_python

4

0

4

benchmarks/tests/test_bench_coding_audit.py

TestFences.test_crlf_fence_still_extracts

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestFences.test_bare_reply_keeps_its_imports

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestFences.test_bare_fallback_does_not_start_inside_prose

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestHarnessCannotBeFakedCheaply.test_printing_the_old_public_marker_no_longer_passes

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestHarnessCannotBeFakedCheaply.test_the_last_marker_line_wins_not_the_first

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestHarnessCannotBeFakedCheaply.test_zero_rows_is_never_a_pass

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestHarnessCannotBeFakedCheaply.test_the_marker_is_a_nonce

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSetupStatements.test_setup_lines_are_not_counted_as_assertions

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSetupStatements.test_a_raising_setup_denies_pass_but_keeps_partial_credit

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSetupStatements.test_helper_defs_are_wrapped_not_counted

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestAstGrouping.test_ordinary_shapes_run_and_score

7

0

7

benchmarks/tests/test_bench_coding_audit.py

TestAstGrouping.test_a_broken_test_string_fails_loudly

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSubprocessHardening.test_the_netns_wrapper_is_applied_when_the_namespace_is_available

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSubprocessHardening.test_main_guard_demo_does_not_run

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSubprocessHardening.test_input_fails_fast_instead_of_waiting_for_the_timeout

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSubprocessHardening.test_timeout_kills_the_whole_process_tree

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestSubprocessHardening.test_unittest_ok_is_not_reported_as_the_failure_reason

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestTruncation.test_server_reported_length_is_a_cut

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestTruncation.test_complete_answer_at_the_cap_is_still_not_forced_to_cut_by_extract

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestAskReasoningDeltas.test_reasoning_only_reply_is_seen_and_finish_length_survives

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestAskReasoningDeltas.test_ollama_style_reasoning_field_is_collected

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestEvaluateAccounting.test_identical_output_is_deterministic_and_counts_tasks

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestEvaluateAccounting.test_agreeing_verdicts_with_different_output_are_not_deterministic

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestEvaluateAccounting.test_cut_attempts_are_excluded_not_failed

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestEvaluateAccounting.test_effective_k_counts_tasks_and_never_rounds_a_ratio

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestEvaluateAccounting.test_a_transport_error_does_not_flip_determinism

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestTaskSets.test_extended_is_the_21_task_set_the_help_describes

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestTaskSets.test_all_is_every_set

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestKvPairsWhitespace.test_whitespace_only_input_must_raise

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCompareUsesObservedCounts.test_effective_k_wins_over_a_rounded_ratio

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCompareUsesObservedCounts.test_legacy_report_without_effective_k_is_clamped_not_trusted

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenTokensAndExits.test_mentioning_sorted_in_a_docstring_is_not_punished

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenTokensAndExits.test_a_docstring_mentioning_an_UNMAPPED_token_is_not_punished

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenTokensAndExits.test_an_unmapped_token_actually_used_is_still_caught

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenTokensAndExits.test_sys_exit_inside_the_function_keeps_earlier_credit

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenOnTheTree.test_evasions_are_caught

6

0

6

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenOnTheTree.test_a_variable_named_sorted_out_is_not_sorted

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenOnTheTree.test_unparseable_code_falls_back_to_the_text_scan

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestUnclosedLastFence.test_a_fence_in_prose_does_not_make_a_typo_a_cut

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestUnclosedLastFence.test_an_unclosed_final_block_is_a_cut

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestRealTokenCount.test_server_usage_overrides_the_delta_count

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestRealTokenCount.test_no_usage_keeps_the_delta_proxy

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestRealTokenCount.test_a_long_legitimate_answer_is_not_cut_by_a_stale_2048

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestRealTokenCount.test_the_thinking_share_uses_the_speed_runners_rule

3

0

3

benchmarks/tests/test_bench_coding_audit.py

TestRealTokenCount.test_ask_reads_completion_tokens

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestStdlibOnly.test_third_party_import_fails_a_stdlib_only_task

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestStdlibOnly.test_stdlib_imports_are_fine

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestStdlibOnly.test_the_prompts_that_say_so_declare_it

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateConfinement.test_environment_is_scrubbed

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestAsciiDigits.test_the_non_ascii_case_is_tested_and_the_reference_passes

2

0

2

benchmarks/tests/test_bench_coding_audit.py

TestAsciiDigits.test_item_list_rejects_non_ascii_and_signed_counts

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestWallClockDeadline.test_a_stream_that_never_ends_is_abandoned

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestWallClockDeadline.test_without_a_deadline_nothing_changes

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestWallClockDeadline.test_evaluate_records_it_unmeasured_not_wrong

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenPrecision.test_the_lookup_evasions_are_still_caught

3

0

3

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenPrecision.test_a_string_value_that_happens_to_read_sort_is_not_a_lookup

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestForbiddenPrecision.test_a_candidate_that_defines_its_own_sort_is_not_using_list_sort

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestInStreamErrors.test_an_openai_style_error_chunk_raises

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestInStreamErrors.test_a_llama_cpp_error_event_line_raises

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestInStreamErrors.test_evaluate_records_errored_not_a_wrong_answer

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestContextOverflow.test_a_4xx_naming_the_context_is_an_overflow

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestContextOverflow.test_an_unrelated_4xx_stays_a_plain_transport_error

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestContextOverflow.test_a_5xx_is_never_an_overflow

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestContextOverflow.test_evaluate_excludes_it_and_records_it_distinctly

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestUnmeasuredWallTime.test_wall_statistics_cover_measured_attempts_only

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestUnmeasuredWallTime.test_the_unmeasured_wall_time_is_still_reported

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestUnmeasuredWallTime.test_a_cut_cannot_flip_the_rank_tiebreak

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateRlimits.test_the_child_runs_under_the_declared_ceilings

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateRlimits.test_the_address_space_ceiling_actually_bites

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateRlimits.test_the_ceiling_counts_tasks_the_census_cannot_see

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateRlimits.test_the_probe_counts_what_the_kernel_charges

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateRlimits.test_the_file_size_ceiling_actually_bites

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestGraderSelfCheck.test_a_failing_reference_aborts_the_run

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestGraderSelfCheck.test_a_passing_reference_records_the_environment

1

0

1

benchmarks/tests/test_bench_coding_audit.py

TestGraderSelfCheck.test_main_stops_before_benchmarking_anything

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestTheSixTasks.test_one_task_per_trap_the_repository_hit

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestExtraction.test_the_defining_fence_wins_over_a_longer_demo

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestExtraction.test_a_prefix_name_does_not_claim_a_longer_function

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestExtraction.test_bare_code_starts_at_the_requires_line

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestAnalyzer.test_a_finding_stops_the_task_before_anything_runs

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestAnalyzer.test_no_pwsh_means_no_analyzer_and_no_probe

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestAnalyzer.test_the_probe_runs_in_the_callers_environment

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestSkips.test_pwsh_absent_is_a_skip_not_a_pass_and_not_a_fail

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestSkips.test_the_self_check_records_pwsh_and_the_analyzer

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_two_passing_rows_are_a_pass

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_more_rows_than_checks_is_refused

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_the_sandbox_is_the_one_the_runner_promises

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_a_pwsh_that_cannot_start_is_reported_not_scored

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_an_analyzer_that_cannot_run_is_a_visible_skip

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_an_analyzer_finding_fails_the_task

1

0

1

benchmarks/tests/test_bench_coding_powershell.py

TestPlumbingWithAStub.test_the_report_row_carries_the_analyzer_note

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

test_reference_solution_passes

33

6

39

benchmarks/tests/test_bench_coding_tasks.py

test_wrong_solution_is_rejected

33

6

39

benchmarks/tests/test_bench_coding_tasks.py

TestTaskShape.test_prompt_states_the_exact_signature

38

1

39

benchmarks/tests/test_bench_coding_tasks.py

TestTaskShape.test_tests_are_not_empty

39

0

39

benchmarks/tests/test_bench_coding_tasks.py

TestTaskShape.test_task_names_are_unique

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestTaskShape.test_novel_tasks_do_not_duplicate_classic_ones

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestTaskTags.test_kind_and_lang_are_explicit_and_known

39

0

39

benchmarks/tests/test_bench_coding_tasks.py

TestTaskTags.test_the_languages_of_this_repository_are_covered

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestTaskTags.test_more_than_one_kind_is_represented

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestTaskTags.test_non_python_tasks_name_the_symbol_or_are_structural

12

0

12

benchmarks/tests/test_bench_coding_tasks.py

TestTaskTags.test_shell_style_checks_are_one_per_line

11

0

11

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_a_correct_function_passes_and_names_shellcheck

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_a_wrong_function_fails_with_partial_credit

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_rows_the_candidate_forged_are_refused

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_an_infinite_loop_is_killed

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_a_script_that_dies_halfway_keeps_the_checks_it_reached

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_the_candidate_runs_with_pipefail

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_an_unset_variable_is_reported_not_silently_empty

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_empty_code_is_not_run

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_the_defining_fence_wins_over_a_longer_demo

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestBashRunner.test_an_unknown_language_is_refused_loudly

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestVisibleSkips.test_cmake_absent_is_a_skip_not_a_pass_and_not_a_fail

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestVisibleSkips.test_shellcheck_absent_is_named_in_the_result_not_swallowed

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestVisibleSkips.test_hadolint_absent_still_grades_the_structure

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestVisibleSkips.test_a_failing_linter_fails_the_task

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestVisibleSkips.test_the_grader_self_check_lists_what_it_could_not_check

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestCmakeRunnerPlumbing.test_two_passing_rows_are_a_pass

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestCmakeRunnerPlumbing.test_a_failing_row_is_reported_with_its_message

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestCmakeRunnerPlumbing.test_more_rows_than_checks_is_refused

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestCmakeRunnerPlumbing.test_a_script_error_is_reported_not_scored

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestDockerfileStructure.test_a_continuation_is_one_instruction

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestDockerfileStructure.test_comments_and_blank_lines_are_not_instructions

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestDockerfileStructure.test_a_dockerfile_reply_is_not_graded_on_its_build_command

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestDockerfileStructure.test_the_task_rejects_a_split_apt_run

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestPerKindAndPerLangRates.test_rates_split_by_lang

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestPerKindAndPerLangRates.test_a_skipped_row_is_not_a_miss

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestPerKindAndPerLangRates.test_rates_split_by_kind

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestPerKindAndPerLangRates.test_an_errored_row_is_excluded_from_both

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestEvaluateExcludesSkips.test_a_skipped_task_is_out_of_the_denominator

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestEvaluateExcludesSkips.test_a_skipped_task_carries_no_measured_wall

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestEvaluateExcludesSkips.test_the_row_says_SKIP_and_why

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestEvaluateExcludesSkips.test_the_report_splits_the_rate_by_lang

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestTaskSetSelection.test_the_default_is_every_task

1

0

1

benchmarks/tests/test_bench_coding_tasks.py

TestTaskSetSelection.test_every_set_name_still_selects_its_own_set

5

0

5

benchmarks/tests/test_bench_coding_tasks.py

TestTaskSetSelection.test_an_unknown_set_is_refused

1

0

1

benchmarks/tests/test_bench_coding_thinking.py

TestACutReplyWithNoMarkerHasNoShare.test_the_row_records_null_and_says_why

1

0

1

benchmarks/tests/test_bench_coding_thinking.py

TestACutReplyWithNoMarkerHasNoShare.test_the_log_line_prints_a_question_mark_not_zero

1

0

1

benchmarks/tests/test_bench_coding_thinking.py

TestACutReplyWithNoMarkerHasNoShare.test_a_deadline_is_a_cut_too

1

0

1

benchmarks/tests/test_bench_coding_thinking.py

TestACutReplyWithNoMarkerHasNoShare.test_a_reply_cut_at_the_budget_is_unknown_even_when_it_passed

1

0

1

benchmarks/tests/test_bench_coding_thinking.py

TestAFinishedReplyWithNoMarkerIsNoThinking.test_the_row_records_zero_and_no_note

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestWhichTasksCarryParaphrases.test_every_classic_and_every_novel_task_has_them

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestWhichTasksCarryParaphrases.test_at_least_four_more_across_the_other_sets

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestWhichTasksCarryParaphrases.test_each_paraphrase_is_a_distinct_rewording

10

0

10

benchmarks/tests/test_bench_coding_variants.py

TestEveryParaphraseIsTheSameTask.test_the_signature_survives_verbatim

16

0

16

benchmarks/tests/test_bench_coding_variants.py

TestEveryParaphraseIsTheSameTask.test_every_literal_survives

16

0

16

benchmarks/tests/test_bench_coding_variants.py

TestEveryParaphraseIsTheSameTask.test_every_stated_constraint_word_survives

16

0

16

benchmarks/tests/test_bench_coding_variants.py

TestEveryParaphraseIsTheSameTask.test_every_worked_example_is_stated

16

0

16

benchmarks/tests/test_bench_coding_variants.py

TestEveryParaphraseIsTheSameTask.test_the_examples_are_the_prompts_own

10

0

10

benchmarks/tests/test_bench_coding_variants.py

TestEveryParaphraseIsTheSameTask.test_a_prompt_with_a_worked_example_declares_it

10

0

10

benchmarks/tests/test_bench_coding_variants.py

TestTheContractHoldsUnderEveryPhrasing.test_the_worked_example_holds_for_the_reference

8

0

8

benchmarks/tests/test_bench_coding_variants.py

TestTheContractHoldsUnderEveryPhrasing.test_the_reference_passes

16

0

16

benchmarks/tests/test_bench_coding_variants.py

TestTheContractHoldsUnderEveryPhrasing.test_the_known_wrong_solution_fails

16

0

16

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_the_spread_and_the_score_per_phrasing_are_reported

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_paraphrases_do_not_inflate_effective_n

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_the_effective_count_observes_each_task_as_written

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_determinism_is_voted_per_phrasing

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_sampling_repeats_count_rounds_not_phrasings

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_phrasings_are_never_identical_follow_ups

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_without_the_flag_nothing_changes

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestEvaluateAsksEveryPhrasing.test_an_errored_phrasing_leaves_the_denominator

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestPhrasingSchedule.test_every_phrasing_once_per_round

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestPhrasingSchedule.test_the_flag_off_asks_the_prompt_as_written

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestMainWiring.test_the_sample_is_net_of_the_suspect_task_and_counted_by_task

1

0

1

benchmarks/tests/test_bench_coding_variants.py

TestMainWiring.test_no_flag_leaves_the_config_as_it_was

1

0

1

benchmarks/tests/test_bench_compare.py

TestNormalisation.test_reads_the_new_envelope

1

0

1

benchmarks/tests/test_bench_compare.py

TestNormalisation.test_reads_the_older_openai_api_envelope

1

0

1

benchmarks/tests/test_bench_compare.py

TestNormalisation.test_the_older_envelope_prefers_its_provenance_block

1

0

1

benchmarks/tests/test_bench_compare.py

TestNormalisation.test_a_run_without_a_correctness_probe_has_no_score

1

0

1

benchmarks/tests/test_bench_compare.py

TestRegressionDetection.test_a_large_drop_is_a_regression

1

0

1

benchmarks/tests/test_bench_compare.py

TestRegressionDetection.test_a_drop_the_sample_cannot_support_is_NOT_a_regression

1

0

1

benchmarks/tests/test_bench_compare.py

TestRegressionDetection.test_an_improvement_is_not_flagged

1

0

1

benchmarks/tests/test_bench_compare.py

TestRegressionDetection.test_unchanged_scores_are_reported_as_such

1

0

1

benchmarks/tests/test_bench_compare.py

TestTiming.test_a_large_slowdown_is_a_regression

1

0

1

benchmarks/tests/test_bench_compare.py

TestTiming.test_small_timing_noise_is_tolerated

1

0

1

benchmarks/tests/test_bench_compare.py

TestTiming.test_the_tolerance_is_adjustable

1

0

1

benchmarks/tests/test_bench_compare.py

TestProvenanceGuards.test_a_changed_grader_is_surfaced

1

0

1

benchmarks/tests/test_bench_compare.py

TestProvenanceGuards.test_comparing_different_benchmarks_is_refused

1

0

1

benchmarks/tests/test_bench_compare.py

TestEntrySetChanges.test_a_disappeared_model_is_reported

1

0

1

benchmarks/tests/test_bench_compare.py

TestEntrySetChanges.test_a_new_model_is_reported_without_failing

1

0

1

benchmarks/tests/test_bench_compare.py

TestRoundTrip.test_loads_a_report_written_to_disk

1

0

1

benchmarks/tests/test_bench_compare.py

TestPerCaseComparison.test_a_flipped_case_is_a_regression_even_when_the_aggregate_is_not

1

0

1

benchmarks/tests/test_bench_compare.py

TestPerCaseComparison.test_the_broken_cases_are_named

1

0

1

benchmarks/tests/test_bench_compare.py

TestPerCaseComparison.test_fixed_cases_are_reported_without_failing

1

0

1

benchmarks/tests/test_bench_compare.py

TestPerCaseComparison.test_a_swap_still_regresses_even_at_an_identical_score

1

0

1

benchmarks/tests/test_bench_compare.py

TestPerCaseComparison.test_new_cases_absent_from_the_baseline_are_ignored

1

0

1

benchmarks/tests/test_bench_compare.py

TestPerCaseComparison.test_reports_without_per_case_detail_still_compare

1

0

1

benchmarks/tests/test_bench_compare.py

TestRepeatsAreNotCollapsed.test_one_flaky_draw_does_not_fire_the_alarm

1

0

1

benchmarks/tests/test_bench_compare.py

TestRepeatsAreNotCollapsed.test_a_total_per_case_collapse_is_caught

1

0

1

benchmarks/tests/test_bench_compare.py

TestRepeatsAreNotCollapsed.test_transport_errors_are_excluded_from_the_case_view

1

0

1

benchmarks/tests/test_bench_compare.py

TestConfigIsCompared.test_a_changed_system_prompt_is_called_out

1

0

1

benchmarks/tests/test_bench_compare.py

TestConfigIsCompared.test_timing_is_not_compared_across_a_repeats_change

1

0

1

benchmarks/tests/test_bench_compare.py

TestTimingUsesPerAttempt.test_a_run_that_errored_out_is_not_reported_as_faster

1

0

1

benchmarks/tests/test_bench_compare.py

TestIntervalsUseEffectiveN.test_repeats_on_a_deterministic_lane_do_not_manufacture_significance

1

0

1

benchmarks/tests/test_bench_compare.py

TestDirectoryPairing.test_pairs_by_file_name

1

0

1

benchmarks/tests/test_bench_compare.py

TestDirectoryPairing.test_manifest_is_an_index_not_a_report

1

0

1

benchmarks/tests/test_bench_compare.py

TestDirectoryPairing.test_a_real_regression_in_any_config_fails

1

0

1

benchmarks/tests/test_bench_compare.py

TestDirectoryPairing.test_identical_runs_do_not_cry_wolf

1

0

1

benchmarks/tests/test_bench_compare.py

TestDirectoryPairing.test_a_vanished_config_is_reported_not_swallowed

1

0

1

benchmarks/tests/test_bench_compare.py

TestDirectoryPairing.test_no_common_reports_is_an_error_not_a_pass

1

0

1

benchmarks/tests/test_bench_compare.py

TestPairedAggregate.test_six_one_way_flips_regress_where_the_intervals_still_overlap

1

0

1

benchmarks/tests/test_bench_compare.py

TestPairedAggregate.test_the_same_aggregate_without_per_case_detail_falls_back_to_overlap

1

0

1

benchmarks/tests/test_bench_compare.py

TestPairedAggregate.test_three_one_way_flips_are_not_separable

1

0

1

benchmarks/tests/test_bench_compare.py

TestPairedAggregate.test_a_swap_of_equal_size_is_a_tie_not_a_regression

1

0

1

benchmarks/tests/test_bench_compare.py

TestPairedAggregate.test_an_improvement_is_marked_separable_by_the_paired_test

1

0

1

benchmarks/tests/test_bench_compare.py

TestPairedAggregate.test_the_difference_interval_is_printed

1

0

1

benchmarks/tests/test_bench_compare.py

TestSingleDrawFlips.test_one_flip_is_reported_not_alarmed

1

0

1

benchmarks/tests/test_bench_compare.py

TestSingleDrawFlips.test_a_deterministic_lane_still_treats_a_flip_as_a_regression

1

0

1

benchmarks/tests/test_bench_compare.py

TestSingleDrawFlips.test_a_provenance_probe_that_saw_two_draws_agree_counts_as_deterministic

1

0

1

benchmarks/tests/test_bench_compare.py

TestSingleDrawFlips.test_a_probe_that_saw_disagreement_does_not

1

0

1

benchmarks/tests/test_bench_compare.py

TestSingleDrawFlips.test_repeats_above_one_keep_the_never_passes_now_rule

1

0

1

benchmarks/tests/test_bench_compare.py

TestContextBlockedRows.test_a_blocked_task_neither_breaks_nor_fixes

1

0

1

benchmarks/tests/test_bench_compare.py

TestContextBlockedRows.test_a_blocked_flag_is_honoured_too

1

0

1

benchmarks/tests/test_bench_compare.py

TestContextBlockedRows.test_blocked_wall_does_not_decide_slower

1

0

1

benchmarks/tests/test_bench_compare.py

TestMeasuredWall.test_wall_measured_s_is_preferred_over_the_total

1

0

1

benchmarks/tests/test_bench_compare.py

TestMeasuredWall.test_falling_back_to_the_total_is_noted

1

0

1

benchmarks/tests/test_bench_compare.py

TestMeasuredWall.test_measured_rows_carry_no_legacy_note

1

0

1

benchmarks/tests/test_bench_compare.py

TestDuplicateLabels.test_normalise_refuses_colliding_labels

1

0

1

benchmarks/tests/test_bench_compare.py

TestDuplicateLabels.test_load_names_the_file

1

0

1

benchmarks/tests/test_bench_compare.py

TestDuplicateLabels.test_distinct_labels_for_the_same_model_are_fine

1

0

1

benchmarks/tests/test_bench_compare.py

TestNotAReport.test_a_json_that_is_neither_envelope_is_refused

1

0

1

benchmarks/tests/test_bench_compare.py

TestSuspectCases.test_control_is_matched_by_backend_even_when_labelled_by_model

1

0

1

benchmarks/tests/test_bench_compare.py

TestSuspectCases.test_control_is_matched_by_a_label_prefix

1

0

1

benchmarks/tests/test_bench_compare.py

TestSuspectCases.test_no_control_yields_none

1

0

1

benchmarks/tests/test_bench_compare.py

TestSuspectCases.test_normalise_carries_suspect_cases

1

0

1

benchmarks/tests/test_bench_compare.py

TestSuspectCases.test_compare_names_them_and_annotates_a_broken_suspect

1

0

1

benchmarks/tests/test_bench_compare.py

TestLanesReports.test_a_large_throughput_drop_regresses

1

0

1

benchmarks/tests/test_bench_compare.py

TestLanesReports.test_a_small_drop_is_noise

1

0

1

benchmarks/tests/test_bench_compare.py

TestLanesReports.test_losing_batching_regresses

1

0

1

benchmarks/tests/test_bench_compare.py

TestLanesReports.test_gaining_batching_does_not

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestTheAggregateOfAnotherLaneSet.test_a_dropped_lane_leaves_the_aggregate_unjudged

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestTheAggregateOfAnotherLaneSet.test_an_added_lane_leaves_the_lane_it_joined_unjudged

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestTheAggregateOfAnotherLaneSet.test_with_every_rate_unjudged_nothing_was_compared

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestTheAggregateOfAnotherLaneSet.test_the_same_lanes_still_judge_the_aggregate

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestEachLaneRuntimeIsDiffed.test_a_rebuilt_lane_is_named_and_an_unchanged_one_says_nothing

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestEachLaneRuntimeIsDiffed.test_other_serve_flags_on_one_lane_are_named

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestEachLaneRuntimeIsDiffed.test_other_weights_behind_the_same_id_are_named

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestEachLaneRuntimeIsDiffed.test_identical_lanes_say_nothing

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestEachLaneRuntimeIsDiffed.test_an_unreadable_runtime_is_no_evidence_of_a_rebuild

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestALaneOnOneSideOnly.test_a_lane_gone_from_the_new_run_is_a_finding

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestALaneOnOneSideOnly.test_a_lane_new_in_this_run_is_a_finding

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestReportsOlderThanTheField.test_the_first_lane_takes_the_provenance_runtime

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestReportsOlderThanTheField.test_an_old_report_against_a_new_one_diffs_what_both_recorded

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestReportsOlderThanTheField.test_the_block_is_the_batching_endpoints_under_batching

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestReportsOlderThanTheField.test_a_lane_nobody_attributed_is_not_given_the_envelopes_runtime

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestOnlyLaneRowsAreLanes.test_the_aggregate_and_batching_rows_are_no_lane

1

0

1

benchmarks/tests/test_bench_compare_lanes.py

TestOnlyLaneRowsAreLanes.test_other_benchmarks_have_no_lane_runtimes

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_a_busy_new_run_withholds_the_timing_verdict

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_the_note_that_decided_it_is_still_printed

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_runs_under_different_load_withhold_it

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_the_spread_of_two_quiet_runs_is_judged

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_a_baseline_that_predates_the_record_is_not_refused_for_it

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_but_a_busy_run_is_refused_against_one

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheGateIsTheLoadNote.test_two_equally_busy_runs_are_refused_for_the_busy_host

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_an_unchanged_time_is_withheld_too

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_faster_against_a_busy_baseline_is_not_called_faster

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_a_score_regression_is_still_judged

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_lane_throughput_is_withheld

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_losing_batching_is_still_judged

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_the_speed_tripwire_is_withheld_on_a_cpu_lane

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_and_where_the_lane_share_is_unknown

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_a_withheld_better_is_not_called_better

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_the_npu_lane_is_still_judged

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestWhatIsWithheld.test_one_side_of_unknown_share_spares_nothing

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheNpuSpare.test_not_past_the_load_it_was_measured_unmoved_at

2

0

2

benchmarks/tests/test_bench_compare_load.py

TestTheNpuSpare.test_a_baseline_that_predates_the_record_is_bounded_by_the_other_side

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheNpuSpare.test_a_spared_verdict_says_why_it_was_judged

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestTheNpuSpare.test_nothing_is_said_of_a_spare_the_gate_did_not_need

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestAllowLoadDifference.test_it_judges_as_before_and_still_prints_the_note

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestExitPrecedence.test_the_order

5

0

5

benchmarks/tests/test_bench_compare_load.py

TestExitPrecedence.test_a_withheld_verdict_alone_is_conditions_differ

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestExitPrecedence.test_a_regression_beside_it_still_exits_1_and_names_both

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestExitPrecedence.test_nothing_compared_on_a_busy_host_stays_nothing_compared

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestExitPrecedence.test_the_flag_restores_todays_verdict

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestExitPrecedence.test_quiet_runs_that_agree_exit_0

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestDirectories.test_a_regression_anywhere_wins

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestDirectories.test_a_withheld_pairing_beats_a_passing_one

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestDirectories.test_and_a_blind_one

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestDirectories.test_all_blind_is_still_nothing_compared

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestDirectories.test_the_flag_judges_every_pairing

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestDirectories.test_a_caller_that_predates_the_flag_is_gated

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestARateItsRequestsLoadLeftUnjudged.test_it_exits_4_and_says_what_the_flag_does_with_it

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestARateItsRequestsLoadLeftUnjudged.test_beside_a_gated_verdict_the_flag_line_names_both

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestARateItsRequestsLoadLeftUnjudged.test_a_metric_that_never_alarms_withholds_nothing

1

0

1

benchmarks/tests/test_bench_compare_load.py

TestARateItsRequestsLoadLeftUnjudged.test_the_manifest_legend_gives_the_same_reasons

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestTheScore.test_an_old_report_scores_its_two_arithmetic_items

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestTheScore.test_its_capability_answers_ride_along_unscored

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestTheScore.test_a_new_report_scores_every_integrity_item

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestTheScore.test_a_cut_integrity_answer_was_not_measured

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestTheScore.test_an_errored_integrity_answer_was_not_measured_either

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestTheScore.test_a_block_without_items_is_scored_whole_as_before

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestOldAgainstNew.test_capability_misses_on_both_sides_are_no_regression

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestOldAgainstNew.test_an_integrity_item_that_broke_is_named_with_the_probe_remedy

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestOldAgainstNew.test_a_capability_move_is_listed_not_judged

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestNewAgainstNew.test_every_integrity_answer_lost_is_a_regression

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestNewAgainstNew.test_every_capability_answer_lost_is_not

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestNewAgainstNew.test_a_capability_answer_lost_is_named

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestACollapse.test_a_collapse_against_an_old_report_is_a_regression

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestACollapse.test_five_of_six_lost_between_new_reports_is_one

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestACollapse.test_one_integrity_miss_is_named_not_judged

1

0

1

benchmarks/tests/test_bench_compare_probe.py

TestACollapse.test_a_lane_broken_on_both_sides_is_no_new_regression

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_thirteen_percent_decode_drop_is_slower

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_run_to_run_noise_is_not

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_prompt_without_a_rate_leaves_the_pairing

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_burst_rate_an_older_report_stored_leaves_the_pairing

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_loss_against_the_tracked_t8_run_is_judged

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_loaded_cpu_lane_is_reported_not_judged

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_energy_is_reported_as_a_ratio_of_sums

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_speed_reports_get_no_latency_verdict

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_loaded_OLD_run_does_not_hide_a_slower_new_one

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_faster_against_a_loaded_old_run_is_not_judged

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_flat_against_a_loaded_old_run_is_not_judged

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_a_decode_verdict_its_requests_left_unjudged_is_withheld

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_the_override_keeps_the_old_reporting

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSpeedTripwire.test_older_reports_have_their_load_derived

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestNothingCompared.test_contract_reports_point_at_their_own_diff

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestNothingCompared.test_reports_with_nothing_in_common_are_not_a_pass

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestGraderSelfcheckNoise.test_timing_and_the_process_ceiling_are_not_a_config_change

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestLaneThroughputIsLikeForLike.test_a_delivered_aggregate_is_compared_with_a_sum_only_as_sums

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSuspectCasesKeepTheUnit.test_agreeing_repeats_stay_one_observation_per_case

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSuspectWallLeavesTheTiming.test_every_wall_statistic_the_producer_wrote_is_recomputed

1

0

1

benchmarks/tests/test_bench_compare_speed.py

TestSuspectWallLeavesTheTiming.test_the_timing_verdict_no_longer_pays_for_the_suspect_case

1

0

1

benchmarks/tests/test_bench_compare_split.py

TestEveryMovedNameIsServedFromBenchCompare.test_a_suspect_case_name

8

0

8

benchmarks/tests/test_bench_compare_split.py

TestEveryMovedNameIsServedFromBenchCompare.test_a_directory_name

3

0

3

benchmarks/tests/test_bench_compare_split.py

TestEveryMovedNameIsServedFromBenchCompare.test_the_baselines_stay_where_save_baseline_wrote_them

1

0

1

benchmarks/tests/test_bench_compare_split.py

TestTheSplitModulesAreLeaves.test_no_reach_back

2

0

2

benchmarks/tests/test_bench_compare_split.py

TestTheSplitModulesAreLeaves.test_each_imports_first_in_a_fresh_interpreter

2

0

2

benchmarks/tests/test_bench_compare_split.py

TestTheSplitModulesAreLeaves.test_a_dir_run_as_a_script_loads_one_copy

1

0

1

benchmarks/tests/test_bench_compare_split.py

TestTheDirectoryLoopIsJudgedByBenchCompare.test_a_patched_compare_decides_the_exit

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestClusteredIntervalBesideTheScore.test_mixed_repeats_print_the_case_clustered_interval

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestClusteredIntervalBesideTheScore.test_repeats_that_agree_keep_todays_display

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestClusteredIntervalBesideTheScore.test_single_draws_keep_todays_display

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestClusteredIntervalBesideTheScore.test_the_interval_covers_the_cases_behind_the_score

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPairedDifferenceInterval.test_identical_per_case_outcomes_are_zero_wide

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPairedDifferenceInterval.test_a_paired_drop_is_estimated_per_case

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPairedDifferenceInterval.test_reports_without_cases_keep_the_unpaired_interval

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_unchanged_is_followed_by_the_drop_it_could_have_missed

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_an_observed_back_flip_rate_is_used

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_the_closing_verdict_states_it

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_the_directory_verdict_states_it

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_a_regression_carries_no_such_line

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_the_closing_line_names_the_weakest_pairing_by_its_drop

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestMinimumDetectableDrop.test_a_reused_seen_holds_only_the_last_comparison

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPassK.test_repeats_on_a_sampling_lane_print_pass_k

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPassK.test_a_deterministic_lane_prints_none

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPassK.test_single_draws_print_none

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPassK.test_different_repeats_compare_at_the_smaller_k

1

0

1

benchmarks/tests/test_bench_compare_stats.py

TestPassK.test_paraphrases_are_not_draws

1

0

1

benchmarks/tests/test_bench_compare_variants.py

TestTheVariantSampleHasOneOwner.test_three_phrasings_asked_once

2

0

2

benchmarks/tests/test_bench_compare_variants.py

TestTheVariantSampleHasOneOwner.test_two_phrasings_over_two_rounds

3

0

3

benchmarks/tests/test_bench_compare_variants.py

TestTheVariantSampleHasOneOwner.test_the_control_keeps_its_rows_and_is_not_rescored

1

0

1

benchmarks/tests/test_bench_compare_variants.py

TestReportsWithoutVariantsKeepTheOldRule.test_the_tracked_reports_recount_as_before

3

0

3

benchmarks/tests/test_bench_compare_variants.py

TestReportsWithoutVariantsKeepTheOldRule.test_a_flag_with_no_paraphrase_keeps_the_old_rule

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestCosine.test_identical_vectors_are_one

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestCosine.test_orthogonal_vectors_are_zero

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestCosine.test_opposite_vectors_are_minus_one

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestCosine.test_scale_does_not_matter

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestCosine.test_a_zero_vector_does_not_divide_by_zero

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestShapeChecks.test_well_formed_vectors_have_no_problems

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestShapeChecks.test_no_vectors_is_reported

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestShapeChecks.test_inconsistent_dimensions_are_caught

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestShapeChecks.test_non_finite_values_are_caught

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestShapeChecks.test_an_all_zero_vector_is_caught

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestTripleQuality.test_every_triple_has_three_distinct_texts

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestTripleQuality.test_there_are_enough_triples_to_mean_something

1

0

1

benchmarks/tests/test_bench_embeddings.py

TestTripleQuality.test_related_shares_vocabulary_less_than_it_shares_meaning

1

0

1

benchmarks/tests/test_bench_integration.py

TestEveryRequestGoesThroughPostJson.test_coding_ask_sends_the_entry_auth_headers_and_extras

1

0

1

benchmarks/tests/test_bench_integration.py

TestEveryRequestGoesThroughPostJson.test_tools_call_sends_the_entry_auth_headers_and_extras

1

0

1

benchmarks/tests/test_bench_integration.py

TestEveryRequestGoesThroughPostJson.test_tools_call_multi_sends_the_entry_auth

1

0

1

benchmarks/tests/test_bench_integration.py

TestEveryRequestGoesThroughPostJson.test_no_entry_still_works_and_sends_no_auth

1

0

1

benchmarks/tests/test_bench_integration.py

TestEveryRequestGoesThroughPostJson.test_an_unset_api_key_variable_is_loud_before_the_request

2

0

2

benchmarks/tests/test_bench_integration.py

TestEveryRequestGoesThroughPostJson.test_the_coding_deadline_still_abandons_the_stream

1

0

1

benchmarks/tests/test_bench_integration.py

TestReportsRecordTheEntryNeverTheKey.test_entry_config_carries_names_and_extras_only

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_a_case_the_control_fails_is_excluded_from_every_candidate

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_the_suspect_cases_are_listed_on_every_row

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_a_case_the_control_only_errored_on_is_not_suspect

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_an_overflow_or_skipped_row_is_not_measured

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_without_a_control_nothing_is_excluded

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_a_control_is_recognised_by_its_label_too

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_a_deterministic_row_keeps_counting_tasks_not_attempts

1

0

1

benchmarks/tests/test_bench_integration.py

TestSuspectCasesLeaveTheRanking.test_bench_tools_rows_key_on_case_and_variant

1

0

1

benchmarks/tests/test_bench_integration.py

TestMainWiring.test_bench_coding_main_excludes_control_failures_and_records_the_entry

1

0

1

benchmarks/tests/test_bench_integration.py

TestMainWiring.test_bench_tools_main_excludes_control_failures_and_records_the_entry

1

0

1

benchmarks/tests/test_bench_integration.py

TestTheSuiteStaysOffline.test_a_socket_connect_inside_a_test_is_refused

1

0

1

benchmarks/tests/test_bench_integration.py

TestTheSuiteStaysOffline.test_an_unpatched_request_fails_loudly_rather_than_hanging

1

0

1

benchmarks/tests/test_bench_integration.py

TestTheRowNamesItsBackend.test_bench_coding_evaluate_records_the_backend

1

0

1

benchmarks/tests/test_bench_integration.py

TestTheRowNamesItsBackend.test_bench_tools_evaluate_records_the_backend

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestEveryRequestKeepsTheBackendEntry.test_every_ask_receives_the_same_entry_object

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestEveryRequestKeepsTheBackendEntry.test_it_survives_repeats_within_one_task

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestForbiddenNamesResolvePerScope.test_binding_the_name_elsewhere_does_not_whitelist_the_call

6

0

6

benchmarks/tests/test_bench_review_fixes.py

TestForbiddenNamesResolvePerScope.test_a_candidates_own_helper_is_still_not_a_use

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestForbiddenNamesResolvePerScope.test_a_local_name_that_is_never_called_is_not_a_use

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestBashCandidateCannotSilenceTheReporter.test_a_correct_answer_with_its_own_exit_trap_still_passes

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestBashCandidateCannotSilenceTheReporter.test_a_wrong_answer_with_its_own_exit_trap_is_still_graded_wrong

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheLinterSkipReachesEveryRow.test_a_failing_row_names_the_absent_linter

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheLinterSkipReachesEveryRow.test_a_passing_row_still_names_it

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheLinterSkipReachesEveryRow.test_the_row_carries_the_linter_verdict

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestThrottlingIsNotAContextOverflow.test_a_throttled_lane_is_not_an_overflow

5

0

5

benchmarks/tests/test_bench_review_fixes.py

TestThrottlingIsNotAContextOverflow.test_a_real_overflow_is_still_one

3

0

3

benchmarks/tests/test_bench_review_fixes.py

TestThrottlingIsNotAContextOverflow.test_a_429_is_recorded_errored_not_overflow

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestSuspectCasesNeedEveryControlAttempt.test_a_case_the_control_solved_once_is_not_suspect

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestSuspectCasesNeedEveryControlAttempt.test_a_case_the_control_never_solved_is_still_suspect

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestSuspectCasesNeedEveryControlAttempt.test_one_bad_wording_does_not_delete_every_paraphrase

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestSuspectExclusionRewritesEveryDerivedTable.test_the_group_tables_follow_the_headline

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestSuspectExclusionRewritesEveryDerivedTable.test_the_wall_no_longer_charges_for_the_suspect_case

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestSuspectExclusionRewritesEveryDerivedTable.test_bench_tools_categories_follow_too

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestAStrayTestFileIsNotACheat.test_a_leftover_scratch_test_is_allowed

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestAStrayTestFileIsNotACheat.test_a_shadowing_copy_of_the_protected_test_is_refused

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestAStrayTestFileIsNotACheat.test_a_conftest_beside_the_protected_test_is_refused

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestAStrayTestFileIsNotACheat.test_a_conftest_far_from_the_protected_test_is_allowed

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestAStrayTestFileIsNotACheat.test_an_extra_check_script_is_allowed

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestAStrayTestFileIsNotACheat.test_another_c_test_is_still_refused_for_the_cmake_fixture

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheReportCarriesTheDeterminismProbe.test_bench_coding_probes_the_lane_it_names

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheReportCarriesTheDeterminismProbe.test_an_unset_api_key_does_not_lose_the_report

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheProbeReachesTheProvenance.test_bench_coding_hands_it_to_write_report

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheProbeReachesTheProvenance.test_bench_tools_hands_it_to_write_report

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestWallMeasuredIsEmitted.test_bench_coding_emits_it

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestWallMeasuredIsEmitted.test_bench_tools_emits_it

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestWallMeasuredIsEmitted.test_bench_compare_reads_it_as_the_preferred_source

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheRankingIsTiered.test_case_outcomes_skips_unmeasured_and_suspect_rows

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheRankingIsTiered.test_two_indistinguishable_rows_print_as_one_tier

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTheRankingIsTiered.test_a_separable_pair_splits_into_two_tiers

1

0

1

benchmarks/tests/test_bench_review_fixes.py

TestTaggedFencesAreNotAutomaticallyInvented.test_quoting_the_tool_error_in_a_tagged_fence_passes

4

0

4

benchmarks/tests/test_bench_review_fixes.py

TestTaggedFencesAreNotAutomaticallyInvented.test_invented_file_contents_still_fail

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchTools.test_the_case_suite_hashes_the_probe_not_the_plumbing

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchTools.test_accept_text_json_hashes_the_same_parser

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchTools.test_prompt_variants_hashes_the_arithmetic_of_the_sample

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchTools.test_a_control_hashes_the_recount_of_every_other_row

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchTools.test_turn_growth_runs_no_probe_and_does_not_hash_it

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchTools.test_turn_growth_beside_a_control_does_not_hash_the_recount

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchEmbeddings.test_takes_a_start_and_hashes_only_itself

1

0

1

benchmarks/tests/test_bench_runstart.py

TestBenchCoding.test_hashes_the_probe_and_the_tasks_not_the_plumbing

5

0

5

benchmarks/tests/test_bench_runstart.py

TestBenchChat.test_a_control_hashes_the_recount_of_every_other_row

1

0

1

benchmarks/tests/test_bench_runstart.py

TestTheSuspectFileJoinsOnlyWithAControl.test_no_control_adds_nothing

1

0

1

benchmarks/tests/test_bench_runstart.py

TestTheSuspectFileJoinsOnlyWithAControl.test_a_control_adds_the_module_itself

2

0

2

benchmarks/tests/test_bench_runstart.py

TestBenchAgent.test_takes_a_start_on_the_opencode_lane

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSlug.test_a_label_becomes_a_filename_fragment

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSlug.test_a_path_separator_cannot_survive

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSlug.test_a_label_of_pure_punctuation_still_yields_a_name

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSlug.test_the_path_is_tool_then_label

1

0

1

benchmarks/tests/test_bench_sweep.py

TestPlanRefusals.test_one_path_per_tool_and_candidate

1

0

1

benchmarks/tests/test_bench_sweep.py

TestPlanRefusals.test_an_existing_file_is_never_overwritten

1

0

1

benchmarks/tests/test_bench_sweep.py

TestPlanRefusals.test_labels_that_slug_to_the_same_file_are_refused

1

0

1

benchmarks/tests/test_bench_sweep.py

TestPlanRefusals.test_the_refusal_happens_before_anything_runs

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_every_known_tool_builds_a_command

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_an_unknown_tool_is_refused

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_coding_gets_the_label_repeats_and_task_set

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_coding_keeps_the_raw_replies

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_the_backend_name_is_passed_when_there_is_one

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_a_url_override_beside_a_backend_is_what_gets_measured

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_an_explicit_url_is_used_when_there_is_no_backend

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_agent_gets_no_endpoint_flags

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_every_command_names_a_file_that_exists

1

0

1

benchmarks/tests/test_bench_sweep.py

TestToolCommands.test_the_tools_command_is_unchanged_by_the_shared_builder

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_chat_is_a_tool_the_sweep_accepts

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_chat_gets_the_endpoint_label_repeats_and_output

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_an_explicit_url_is_used_when_there_is_no_backend

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_chat_keeps_its_own_token_budget_and_every_category

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_every_flag_it_passes_is_one_bench_chat_accepts

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_a_sweep_writes_chat_to_its_own_derived_file

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_an_existing_chat_report_is_never_overwritten

1

0

1

benchmarks/tests/test_bench_sweep.py

TestChatStep.test_an_unreachable_candidate_skips_its_chat_step

1

0

1

benchmarks/tests/test_bench_sweep.py

TestAgentRepeats.test_the_agent_step_gets_the_sweeps_repeats

1

0

1

benchmarks/tests/test_bench_sweep.py

TestAgentRepeats.test_repeats_below_one_are_refused_before_anything_runs

1

0

1

benchmarks/tests/test_bench_sweep.py

TestGate.test_a_full_score_is_ok

1

0

1

benchmarks/tests/test_bench_sweep.py

TestGate.test_a_wrong_answer_is_recorded_as_wrong

1

0

1

benchmarks/tests/test_bench_sweep.py

TestGate.test_truncation_is_not_incorrectness

1

0

1

benchmarks/tests/test_bench_sweep.py

TestGate.test_an_unreachable_endpoint_is_its_own_verdict

1

0

1

benchmarks/tests/test_bench_sweep.py

TestGate.test_only_integrity_answers_decide_it

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_the_gate_runs_before_the_tools

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_an_unreachable_candidate_is_not_measured

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_a_degraded_candidate_is_still_measured_and_recorded

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_the_verdict_is_written_next_to_the_results

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_the_summary_is_hidden_from_the_manifest_glob

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_a_failing_tool_does_not_stop_the_sweep

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_the_manifest_step_runs_last

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_no_comparison_without_a_baseline

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_a_baseline_compares_every_written_report

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_a_report_the_tool_never_wrote_is_not_compared

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_a_regression_is_recorded_not_hidden

1

0

1

benchmarks/tests/test_bench_sweep.py

TestSweep.test_a_withheld_verdict_is_recorded_as_conditions_differ

1

0

1

benchmarks/tests/test_bench_sweep.py

TestMainValidation.test_an_unknown_tool_name_is_refused

1

0

1

benchmarks/tests/test_bench_sweep.py

TestMainValidation.test_an_empty_tool_list_is_refused

1

0

1

benchmarks/tests/test_bench_sweep.py

TestMainValidation.test_a_missing_baseline_is_caught_before_any_measurement

1

0

1

benchmarks/tests/test_bench_sweep.py

TestMainValidation.test_an_empty_candidates_file_is_refused

1

0

1

benchmarks/tests/test_bench_sweep.py

TestMainValidation.test_a_whole_run_writes_the_summary_and_exits_zero

1

0

1

benchmarks/tests/test_bench_sweep.py

TestMainValidation.test_a_failed_tool_makes_the_sweep_exit_non_zero

1

0

1

benchmarks/tests/test_bench_sweep.py

TestASweepThatMeasuredNothingFailsLoudly.test_an_all_skipped_sweep_exits_non_zero

1

0

1

benchmarks/tests/test_bench_sweep.py

TestASweepThatMeasuredNothingFailsLoudly.test_a_sweep_that_measured_something_still_exits_zero

1

0

1

benchmarks/tests/test_bench_sweep.py

TestASweepThatMeasuredNothingFailsLoudly.test_a_failed_step_keeps_its_own_exit_code_and_wording

1

0

1

benchmarks/tests/test_bench_sweep.py

TestTheSweepSummaryHoldsTheArgv.test_every_step_records_its_command

1

0

1

benchmarks/tests/test_bench_sweep.py

TestTheSweepSummaryHoldsTheArgv.test_a_skipped_step_records_the_command_it_would_have_run

1

0

1

benchmarks/tests/test_bench_sweep.py

TestTheSweepSummaryHoldsTheArgv.test_the_argv_survives_into_the_file

1

0

1

benchmarks/tests/test_bench_tools.py

TestHappyPath.test_correct_call_with_json_string_arguments

1

0

1

benchmarks/tests/test_bench_tools.py

TestHappyPath.test_arguments_may_arrive_as_a_dict

1

0

1

benchmarks/tests/test_bench_tools.py

TestHappyPath.test_tolerates_a_leading_dot_slash

1

0

1

benchmarks/tests/test_bench_tools.py

TestHappyPath.test_tool_with_no_required_arguments

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_wrong_tool_is_caught

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_missing_required_argument

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_wrong_argument_value

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_invalid_json_arguments

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_prose_instead_of_a_call

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_several_calls_when_one_was_expected

1

0

1

benchmarks/tests/test_bench_tools.py

TestFailures.test_boolean_argument_must_match_exactly

1

0

1

benchmarks/tests/test_bench_tools.py

TestNoToolExpected.test_answering_directly_is_correct

1

0

1

benchmarks/tests/test_bench_tools.py

TestNoToolExpected.test_calling_a_tool_anyway_is_wrong

1

0

1

benchmarks/tests/test_bench_tools.py

TestSuiteShape.test_every_expected_tool_exists_in_the_advertised_set

1

0

1

benchmarks/tests/test_bench_tools.py

TestSuiteShape.test_the_suite_includes_a_negative_case

1

0

1

benchmarks/tests/test_bench_tools.py

TestSuiteShape.test_case_names_are_unique

1

0

1

benchmarks/tests/test_bench_tools.py

TestSuiteShape.test_the_suite_is_large_enough_to_prove_a_regression

1

0

1

benchmarks/tests/test_bench_tools.py

TestSuiteShape.test_several_tools_are_near_neighbours

1

0

1

benchmarks/tests/test_bench_tools.py

TestSuiteShape.test_restraint_is_measured_more_than_once

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_every_multi_case_has_a_known_grader

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_use_result_cases_state_what_must_appear

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_recover_cases_feed_back_an_actual_error

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_histories_are_well_formed

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_deep_history_case_has_five_tool_turns

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_long_result_case_is_long_and_holds_the_fact_once

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_repeated_error_case_fails_the_same_call_twice

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiCaseShape.test_both_kinds_are_represented

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiTurn.test_uses_the_returned_value

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiTurn.test_ignoring_the_result_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiTurn.test_calling_another_tool_instead_of_answering_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiTurn.test_empty_reply_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestMultiTurn.test_match_is_case_insensitive

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecovery.test_admitting_the_failure_passes

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecovery.test_retrying_with_another_call_passes

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecovery.test_inventing_content_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecovery.test_empty_reply_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_recovers_a_parameters_shaped_call

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_recovers_an_arguments_shaped_call

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_recovers_from_surrounding_prose

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_stringified_arguments_are_parsed

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_a_wrong_tool_in_text_is_still_wrong

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_wrong_arguments_in_text_are_still_wrong

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_prose_without_json_recovers_nothing

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_json_without_a_name_recovers_nothing

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonFallback.test_the_no_tool_case_is_unaffected

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_integer_one_is_not_true

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_string_true_is_not_true

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_string_digits_are_not_an_integer

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_true_is_not_an_integer

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_enum_value_outside_the_set_is_named

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_array_is_order_insensitive_but_typed

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_every_typed_case_uses_a_declared_type

1

0

1

benchmarks/tests/test_bench_tools.py

TestTypedArguments.test_tools_cover_int_bool_enum_and_array

1

0

1

benchmarks/tests/test_bench_tools.py

TestPathLeniency.test_trailing_slash_on_a_directory

1

0

1

benchmarks/tests/test_bench_tools.py

TestPathLeniency.test_a_trailing_dot_on_content_is_wrong

1

0

1

benchmarks/tests/test_bench_tools.py

TestPathLeniency.test_a_slashed_query_is_wrong

1

0

1

benchmarks/tests/test_bench_tools.py

TestPathLeniency.test_only_one_dot_slash_is_forgiven

1

0

1

benchmarks/tests/test_bench_tools.py

TestPathLeniency.test_patch_requires_the_diff

1

0

1

benchmarks/tests/test_bench_tools.py

TestParallelCalls.test_both_calls_in_either_order

1

0

1

benchmarks/tests/test_bench_tools.py

TestParallelCalls.test_only_one_of_two_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestParallelCalls.test_the_same_call_twice_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestParallelCalls.test_three_calls_for_two_expected_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestParallelCalls.test_parallel_multiple_mixes_tools

1

0

1

benchmarks/tests/test_bench_tools.py

TestParallelCalls.test_suite_has_parallel_and_parallel_multiple_cases

1

0

1

benchmarks/tests/test_bench_tools.py

TestRestraintEmptyReply.test_empty_none_and_whitespace_all_fail

1

0

1

benchmarks/tests/test_bench_tools.py

TestIrrelevanceAndVariants.test_irrelevance_prompts_never_mention_tools

1

0

1

benchmarks/tests/test_bench_tools.py

TestIrrelevanceAndVariants.test_instructed_restraint_is_still_measured

1

0

1

benchmarks/tests/test_bench_tools.py

TestIrrelevanceAndVariants.test_selection_variants_share_under_two_content_words_with_the_description

1

0

1

benchmarks/tests/test_bench_tools.py

TestIrrelevanceAndVariants.test_every_case_has_a_category

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_identical_retry_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_identical_retry_with_dot_slash_still_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_different_call_passes

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_third_identical_call_in_repeated_error_case_fails

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_admit_words_are_word_bounded

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_quoting_the_tool_output_alone_is_not_an_admission

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_own_admission_beside_a_quote_passes

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_invented_fenced_block_fails_even_with_an_admission

1

0

1

benchmarks/tests/test_bench_tools.py

TestErrorRecoveryHistory.test_history_is_optional

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonInMultiTurn.test_followup_written_as_a_text_call_fails_with_or_without_the_flag

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonInMultiTurn.test_text_retry_passes_recovery_under_the_flag

1

0

1

benchmarks/tests/test_bench_tools.py

TestTextJsonInMultiTurn.test_text_identical_retry_fails_under_the_flag

1

0

1

benchmarks/tests/test_bench_tools.py

TestQwenTemplateSalvage.test_two_blocks_yield_two_calls

1

0

1

benchmarks/tests/test_bench_tools.py

TestQwenTemplateSalvage.test_values_are_coerced_by_the_schema_not_by_shape

1

0

1

benchmarks/tests/test_bench_tools.py

TestQwenTemplateSalvage.test_a_json_list_recovers_several_calls

1

0

1

benchmarks/tests/test_bench_tools.py

TestContextPadding.test_padding_never_contains_the_answer_key

1

0

1

benchmarks/tests/test_bench_tools.py

TestOpencodeToolSet.test_ten_tools_at_a_realistic_length

1

0

1

benchmarks/tests/test_bench_tools.py

TestOpencodeToolSet.test_required_parameters_exist

1

0

1

benchmarks/tests/test_bench_tools.py

TestOpencodeToolSet.test_every_translation_names_an_advertised_tool

1

0

1

benchmarks/tests/test_bench_tools.py

TestOpencodeToolSet.test_translated_history_uses_advertised_tools

1

0

1

benchmarks/tests/test_bench_tools.py

TestOpencodeToolSet.test_translated_patch_grades_an_edit

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAMultiTurnTransportFailureLeavesTheDenominator.test_a_raising_multi_turn_call_is_excluded_not_scored_wrong

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAMultiTurnTransportFailureLeavesTheDenominator.test_the_errored_row_is_flagged_and_carries_no_wall

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAMultiTurnTransportFailureLeavesTheDenominator.test_a_single_turn_transport_failure_is_excluded_too

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAnErroredCaseSaysWhy.test_the_row_keeps_what_it_had_and_adds_status_and_body

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAnErroredCaseSaysWhy.test_the_printed_line_says_it_too

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAnErroredCaseSaysWhy.test_a_single_turn_case_says_it_too

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAnErroredCaseSaysWhy.test_the_body_is_cut_at_500_characters_and_printed_on_one_line

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAnErroredCaseSaysWhy.test_any_other_failure_records_what_it_always_did

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestAnErroredCaseSaysWhy.test_the_score_is_what_any_other_failure_gives

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestTheDeterminismBookkeeping.test_byte_identical_replies_are_deterministic

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestTheDeterminismBookkeeping.test_differing_text_with_the_same_verdict_is_not_deterministic

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestTheDeterminismBookkeeping.test_effective_k_is_a_count_when_attempts_are_uneven

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestTheDeterminismBookkeeping.test_a_single_run_is_never_called_deterministic

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestTheReportSaysWhatChangedTheScore.test_the_flags_that_change_the_score_are_on_the_row

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestTheReportSaysWhatChangedTheScore.test_the_per_case_key_is_case_and_variant

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestNoRegressionAgainstACleanBaseline.test_a_transport_error_is_not_a_regression

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestNoRegressionAgainstACleanBaseline.test_a_case_that_stopped_passing_IS_a_regression

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestRepeatsAreNeverIdenticalFollowUps.test_a_spacer_separates_the_repeats_of_one_case

1

0

1

benchmarks/tests/test_bench_tools_evaluate.py

TestRepeatsAreNeverIdenticalFollowUps.test_one_draw_needs_no_spacer

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestTheRepliesAreTheTrackedOnes.test_the_arithmetic_reply_passed_without_the_flag_and_failed_with_it

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestTheRepliesAreTheTrackedOnes.test_the_listing_reply_is_the_r1_runs

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestARestraintCaseFailsACallWrittenAsText.test_they_are_the_cases_that_expect_none

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestARestraintCaseFailsACallWrittenAsText.test_the_tracked_reply_fails_with_or_without_the_flag

2

0

2

benchmarks/tests/test_bench_tools_text_calls.py

TestARestraintCaseFailsACallWrittenAsText.test_the_verdict_does_not_depend_on_the_flag

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestARestraintCaseFailsACallWrittenAsText.test_a_qwen_template_block_is_a_call_too

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestARestraintCaseFailsACallWrittenAsText.test_an_answer_that_names_no_call_still_passes

3

0

3

benchmarks/tests/test_bench_tools_text_calls.py

TestAFollowUpFailsACallWrittenAsText.test_they_are_the_cases_graded_by_grade_followup

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestAFollowUpFailsACallWrittenAsText.test_the_tracked_listing_reply_fails_on_its_call

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestAFollowUpFailsACallWrittenAsText.test_a_call_that_names_the_fact_is_not_an_answer

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestAFollowUpFailsACallWrittenAsText.test_an_answer_from_the_result_still_passes

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestTheCaseSuiteRowSaysSo.test_a_restraint_row

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestTheCaseSuiteRowSaysSo.test_a_follow_up_row

6

0

6

benchmarks/tests/test_bench_tools_text_calls.py

TestTheRowQuotesWhereTheCallWasRead.test_a_restraint_row

1

0

1

benchmarks/tests/test_bench_tools_text_calls.py

TestTheRowQuotesWhereTheCallWasRead.test_a_follow_up_row

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestAFailedTurnSaysWhy.test_the_row_keeps_the_status_line_and_adds_status_and_body

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestAFailedTurnSaysWhy.test_the_printed_line_says_it_too

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestAFailedTurnSaysWhy.test_the_body_is_cut_at_500_characters

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestAFailedTurnSaysWhy.test_a_multi_line_body_prints_on_one_line

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestAFailedTurnSaysWhy.test_any_other_failure_records_what_it_always_did

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestAFailedTurnSaysWhy.test_the_turns_before_it_are_unchanged

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_two_calls_get_two_answers_in_call_order

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_a_strict_server_refuses_no_turn

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_the_strict_stub_refuses_what_the_old_loop_sent

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_the_strict_stub_reads_every_turn_not_only_the_first

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_the_row_records_how_many_calls_the_turn_made

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_the_line_says_how_many_and_a_single_call_reads_as_before

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_a_call_without_an_id_gets_one_on_both_sides

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_a_turn_is_still_one_request_timed_alone

1

0

1

benchmarks/tests/test_bench_tools_turn_growth.py

TestEveryCallOfATurnIsAnswered.test_every_answer_counts_in_the_context_estimate

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_pass_on_one_phrasing_and_fail_on_another_is_spread

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_agreeing_phrasings_are_not_spread_either_way

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_a_flaky_draw_on_every_phrasing_is_noise_not_wording

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_a_phrasing_that_never_passes_is_spread_on_a_sampling_lane

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_a_case_asked_one_way_cannot_disagree_and_is_not_in_the_rate

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_a_phrasing_with_no_measured_attempt_is_zero_of_zero

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_no_variant_case_gives_no_rate_rather_than_zero

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheSpreadDefinition.test_the_task_key_works_for_bench_coding_rows

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheScorePerPhrasing.test_each_phrasing_is_scored_over_the_cases_asked_more_than_one_way

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestParaphrasesDoNotInflateTheSample.test_one_draw_per_phrasing_is_one_observation_per_case

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestParaphrasesDoNotInflateTheSample.test_agreeing_repeats_collapse_to_the_case

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestParaphrasesDoNotInflateTheSample.test_disagreeing_repeats_keep_one_observation_per_round

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestParaphrasesDoNotInflateTheSample.test_the_observation_is_the_prompt_as_written

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestParaphrasesDoNotInflateTheSample.test_an_unmeasured_v0_is_observed_through_the_next_phrasing

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestParaphrasesDoNotInflateTheSample.test_a_sampling_lane_is_not_charged_once_per_paraphrase

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestPhrasingAgreement.test_the_vote_is_per_phrasing_because_two_phrasings_are_two_prompts

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestPhrasingAgreement.test_differing_text_with_the_same_verdict_agrees_but_is_not_deterministic

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestPhrasingAgreement.test_one_draw_is_never_called_deterministic

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestThePrintedLines.test_the_spread_the_per_phrasing_score_and_the_sample_are_printed

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestThePrintedLines.test_one_draw_per_phrasing_says_the_spread_may_be_the_sampler

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestThePrintedLines.test_a_suite_without_paraphrases_says_so

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestThePrintedLines.test_the_unit_is_named_by_the_caller

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_the_row_carries_the_spread_and_the_score_per_phrasing

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_paraphrases_do_not_inflate_effective_n

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_the_effective_count_observes_each_case_as_written

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_deterministic_repeats_collapse_to_cases_not_to_case_variant_pairs

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_sampling_repeats_count_rounds_not_phrasings

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_without_the_flag_nothing_changes

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_a_flag_with_no_paraphrase_to_ask_keeps_the_old_sample

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestEvaluateReportsTheSpread.test_phrasings_are_never_identical_follow_ups

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheControlLeavesTheSampleInOnePass.test_the_suspect_case_leaves_the_sample_and_the_spread

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheControlLeavesTheSampleInOnePass.test_the_control_keeps_its_full_score

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestTheControlLeavesTheSampleInOnePass.test_a_row_without_variants_keeps_the_producers_rule

1

0

1

benchmarks/tests/test_bench_tools_variants.py

TestMainWritesTheSpread.test_the_report_carries_the_spread_net_of_suspect_cases

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestRealServerOutput.test_the_9b_call_is_recovered

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestRealServerOutput.test_the_template_does_not_leak_into_the_text

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestRealServerOutput.test_the_2b_has_no_call_to_recover

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestParsing.test_plain_text_is_untouched_apart_from_thinking

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestParsing.test_several_calls_stay_separate

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestParsing.test_several_parameters_are_all_kept

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestParsing.test_inner_indentation_survives

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestParsing.test_arguments_are_a_json_string_not_an_object

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestParsing.test_a_truncated_call_does_not_leak_markup

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestResponseConversion.test_finish_reason_becomes_tool_calls

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestResponseConversion.test_a_server_that_already_parsed_is_left_alone

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestResponseConversion.test_a_plain_answer_keeps_finish_reason_stop

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestResponseConversion.test_empty_choices_do_not_crash

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestStreamRendering.test_the_stream_carries_the_tool_call_and_terminates

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestThinking.test_paired_block_is_removed

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestThinking.test_unpaired_closing_tag_is_removed

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestThinking.test_a_pair_and_a_stray_closer_in_one_message

1

0

1

benchmarks/tests/test_geniex_toolcall_shim.py

TestThinking.test_an_unclosed_opening_tag_is_left_alone

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_pure_legacy_quant_is_ok

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_three_bit_k_quant_is_ok

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_iq4_is_not_flagged

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_iquant_dominated_is_risky

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_handful_of_iquant_tensors_is_likely_ok

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_metadata_and_counts

1

0

1

benchmarks/tests/test_inspect_gguf.py

TestVerdicts.test_rejects_non_gguf

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_a_tagged_pdf_short_circuits_to_born_digital

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_tagged_wins_even_over_a_degenerate_layer

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_a_page_at_the_text_gate_is_born_digital

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_a_page_just_under_the_text_gate_is_not_born_digital

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_whitespace_does_not_count_toward_the_text_gate

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_a_long_nul_ridden_text_layer_is_degenerate_not_born_digital

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_one_image_covering_the_page_is_image_only

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_an_image_just_under_the_cover_gate_is_not_a_scan

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_many_small_images_do_not_add_up_to_a_scan

1

0

1

benchmarks/tests/test_nas_census.py

TestClassifyPdfPage.test_an_empty_page_is_sparse

1

0

1

benchmarks/tests/test_nas_census.py

TestDegenerateText.test_a_run_of_three_nuls_is_degenerate

1

0

1

benchmarks/tests/test_nas_census.py

TestDegenerateText.test_two_isolated_nuls_in_clean_text_are_not

1

0

1

benchmarks/tests/test_nas_census.py

TestDegenerateText.test_a_replacement_char_majority_is_degenerate

1

0

1

benchmarks/tests/test_nas_census.py

TestDegenerateText.test_the_bad_char_ratio_boundary_is_exclusive

1

0

1

benchmarks/tests/test_nas_census.py

TestDegenerateText.test_clean_german_text_is_not_degenerate

1

0

1

benchmarks/tests/test_nas_census.py

TestDegenerateText.test_empty_text_is_not_degenerate

1

0

1

benchmarks/tests/test_nas_census.py

TestDetectLanguage.test_a_german_paragraph_is_de

1

0

1

benchmarks/tests/test_nas_census.py

TestDetectLanguage.test_an_english_paragraph_is_en

1

0

1

benchmarks/tests/test_nas_census.py

TestDetectLanguage.test_a_tie_decides_nothing

1

0

1

benchmarks/tests/test_nas_census.py

TestDetectLanguage.test_too_little_text_decides_nothing

1

0

1

benchmarks/tests/test_nas_census.py

TestDetectLanguage.test_no_stopwords_decides_nothing

1

0

1

benchmarks/tests/test_nas_census.py

TestTheGate.test_under_ten_percent_scanned_makes_the_vlm_a_footnote

1

0

1

benchmarks/tests/test_nas_census.py

TestTheGate.test_ten_percent_or_more_means_measure_ocr_first

1

0

1

benchmarks/tests/test_nas_census.py

TestTheGate.test_the_gate_flips_exactly_at_ten_percent

1

0

1

benchmarks/tests/test_nas_census.py

TestTheGate.test_no_classified_pages_is_not_a_zero_fraction

1

0

1

benchmarks/tests/test_nas_census.py

TestSampling.test_a_small_pdf_is_read_in_full

1

0

1

benchmarks/tests/test_nas_census.py

TestSampling.test_zero_sample_means_every_page

1

0

1

benchmarks/tests/test_nas_census.py

TestSampling.test_a_large_pdf_yields_evenly_spread_unique_indices

1

0

1

benchmarks/tests/test_nas_census.py

TestSampling.test_extrapolation_preserves_the_sampled_ratio

1

0

1

benchmarks/tests/test_nas_census.py

TestSampling.test_extrapolated_counts_always_sum_to_the_page_total

1

0

1

benchmarks/tests/test_nas_census.py

TestSampling.test_a_fully_read_pdf_is_not_extrapolated

1

0

1

benchmarks/tests/test_nas_census.py

TestOfficeExtraction.test_docx_text_is_extracted

1

0

1

benchmarks/tests/test_nas_census.py

TestOfficeExtraction.test_xlsx_reads_shared_strings_and_inline_strings

1

0

1

benchmarks/tests/test_nas_census.py

TestOfficeExtraction.test_pptx_slide_runs_are_extracted

1

0

1

benchmarks/tests/test_nas_census.py

TestOfficeExtraction.test_a_corrupt_docx_returns_none_not_an_exception

1

0

1

benchmarks/tests/test_nas_census.py

TestOfficeExtraction.test_a_zip_without_the_expected_member_returns_none

1

0

1

benchmarks/tests/test_nas_census.py

TestTheWalker.test_a_corrupt_file_lands_in_errors_and_the_walk_completes

1

0

1

benchmarks/tests/test_nas_census.py

TestTheWalker.test_without_fitz_the_pdf_is_counted_and_classification_skips_visibly

1

0

1

benchmarks/tests/test_nas_census.py

TestTheWalker.test_languages_are_tallied_per_document

1

0

1

benchmarks/tests/test_nas_census.py

TestTheWalker.test_table_density_is_not_measured_without_the_flag

1

0

1

benchmarks/tests/test_nas_census.py

TestTheWalker.test_max_files_truncation_is_loud_in_summary_and_json

1

0

1

benchmarks/tests/test_nas_census.py

TestTheWalker.test_two_runs_over_the_same_tree_walk_identically

1

0

1

benchmarks/tests/test_nas_census.py

TestTheCli.test_a_missing_root_exits_two

1

0

1

benchmarks/tests/test_nas_census.py

TestTheCli.test_json_output_carries_the_four_numbers_and_the_gate

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_a_correct_solution_scores_every_assertion

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_one_missed_edge_case_is_not_the_same_as_total_failure

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_later_assertions_still_run_after_an_early_failure

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_the_detail_names_the_score

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_a_syntax_error_yields_no_assertions_rather_than_a_crash

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_a_forbidden_construct_short_circuits_before_running

1

0

1

benchmarks/tests/test_partial_credit.py

TestPartialCredit.test_every_assertion_is_reported_individually

1

0

1

benchmarks/tests/test_partial_credit.py

TestEnvironmentRecord.test_live_lanes_are_recorded

1

0

1

benchmarks/tests/test_partial_credit.py

TestEnvironmentRecord.test_differing_lanes_are_flagged_when_comparing

1

0

1

benchmarks/tests/test_partial_credit.py

TestEnvironmentRecord.test_an_unreachable_registry_does_not_raise

1

0

1

benchmarks/tests/test_partial_credit.py

TestAssertionGrouping.test_try_except_stays_one_statement

1

0

1

benchmarks/tests/test_partial_credit.py

TestAssertionGrouping.test_each_top_level_assert_is_its_own_group

1

0

1

benchmarks/tests/test_partial_credit.py

TestAssertionGrouping.test_else_and_finally_also_stay_attached

1

0

1

benchmarks/tests/test_partial_credit.py

TestAssertionGrouping.test_comments_do_not_become_their_own_group

1

0

1

benchmarks/tests/test_partial_credit.py

TestShouldRaiseChecksAreAssertions.test_a_should_raise_block_counts_as_exactly_one_assertion

1

0

1

benchmarks/tests/test_partial_credit.py

TestShouldRaiseChecksAreAssertions.test_a_try_that_asserts_nothing_is_still_setup

1

0

1

benchmarks/tests/test_partial_credit.py

TestShouldRaiseChecksAreAssertions.test_the_shipped_task_counts_its_two_should_raise_checks

1

0

1

benchmarks/tests/test_partial_credit.py

TestShouldRaiseChecksAreAssertions.test_missing_one_rule_scores_n_minus_one_not_a_setup_failure

1

0

1

benchmarks/tests/test_partial_credit.py

TestShouldRaiseChecksAreAssertions.test_the_reference_still_scores_every_assertion

1

0

1

benchmarks/tests/test_run_benchmarks_smoke.py

test_run_benchmarks_sh_parses

1

0

1

benchmarks/tests/test_run_benchmarks_smoke.py

test_run_benchmarks_sh_imports_resolve_backend_from_the_package

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_each_lane_runs_its_steps_in_the_protocol_order_then_compare

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_steps_given_out_of_order_still_run_in_order

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_the_commands_are_the_protocols

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_every_script_it_names_exists

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_files_are_named_after_lane_and_step

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_no_correctness_drops_only_the_gate

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_overflow_tokens_reach_only_their_lane

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_a_lane_without_a_previous_report_is_not_diffed

1

0

1

benchmarks/tests/test_upgrade_check.py

TestPlan.test_without_previous_the_comparison_is_skipped_with_a_reason

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_an_existing_out_directory_is_refused_even_when_empty

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_a_previous_that_is_not_a_directory_is_refused

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_an_environment_url_override_is_refused

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_an_unknown_lane_is_refused_naming_the_known_ones

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_a_lane_named_twice_is_refused

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_an_unknown_step_is_refused

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_overflow_for_a_lane_outside_the_check_is_refused

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_wsl_is_refused_where_coding_runs_natively

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRefusals.test_a_dry_run_creates_and_contacts_nothing

1

0

1

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_without_wsl_coding_is_skipped_and_its_command_kept

1

0

1

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_with_wsl_it_runs_in_the_named_distro_on_mnt_paths

1

0

1

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_the_registry_named_on_windows_is_the_one_read_in_wsl

1

0

1

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_a_relative_registry_is_made_absolute_for_the_children

1

0

1

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_a_path_wsl_cannot_reach_is_refused_with_wsl

1

0

1

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_to_wsl_path

3

0

3

benchmarks/tests/test_upgrade_check.py

TestCodingOnWindows.test_a_skipped_coding_step_does_not_fail_the_check

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_clean_run_exits_0_and_logs_every_step

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_only_reports_are_json_in_the_run_directory

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_the_manifest_maps_each_file_to_its_command_and_exit

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_failed_step_fails_the_check_and_the_rest_still_run

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_exit_0_without_the_report_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_contract_that_got_no_answer_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_speed_run_with_no_completed_prompt_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_speed_run_that_lost_a_prompt_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_wrong_answer_from_the_gate_fails_the_speed_step

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_gate_that_scored_nothing_fails_the_speed_step

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_without_the_gate_no_correctness_block_is_expected

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_report_of_a_foreign_shape_fails_its_step_not_the_check

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_check_that_ran_nothing_does_not_pass

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_an_unreachable_lane_is_not_measured

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_lane_relaunched_mid_check_fails_it

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_both_runtime_probes_name_the_lanes_model

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_a_model_re_pulled_mid_check_fails_it

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_unchanged_model_files_leave_the_lane_alone

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_an_upgrade_mid_check_is_named

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_the_manifest_is_rewritten_after_every_step

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRun.test_ctrl_c_keeps_the_record_and_exits_130

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_no_regression_exits_0

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_a_regression_is_its_own_status_and_fails_the_check

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_a_regression_then_a_crash_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_exit_1_without_a_verdict_is_a_failure_not_a_regression

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_exit_3_is_nothing_compared_and_not_a_pass

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_exit_4_is_conditions_differ_and_fails_the_check

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_exit_4_that_did_not_finish_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_a_regression_beside_a_withheld_verdict_is_a_regression

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_no_shared_file_name_is_nothing_compared_without_running_it

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_a_moved_contract_answer_is_listed_not_failed

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_a_diff_that_exits_1_with_nothing_moved_is_a_failure

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_a_diff_that_crashed_is_a_failure_even_when_answers_moved

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_no_diff_after_a_contract_that_got_no_answer

1

0

1

benchmarks/tests/test_upgrade_check.py

TestComparison.test_the_previous_runtime_is_named_beside_the_new_one

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRunLogged.test_output_reaches_the_log_and_the_exit_code_is_returned

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRunLogged.test_a_command_that_cannot_start_exits_127

1

0

1

benchmarks/tests/test_upgrade_check.py

TestRunLogged.test_children_see_the_repo_unbuffered_and_in_utf8

1

0

1

benchmarks/tests/test_upgrade_check.py

TestLaneWarnings.test_a_geniex_lane_logging_at_info_is_flagged

1

0

1

benchmarks/tests/test_upgrade_check.py

TestLaneWarnings.test_nothing_else_is

4

0

4

benchmarks/tests/test_upgrade_check_gate.py

TestTheSpeedStepGate.test_a_capability_miss_passes_the_step

1

0

1

benchmarks/tests/test_upgrade_check_gate.py

TestTheSpeedStepGate.test_an_integrity_miss_fails_it

1

0

1

benchmarks/tests/test_upgrade_check_gate.py

TestTheSpeedStepGate.test_an_old_report_fails_only_on_its_arithmetic

1

0

1

benchmarks/tests/test_upgrade_check_gate.py

TestTheSpeedStepGate.test_a_block_without_items_is_judged_whole_as_before

1

0

1

benchmarks/tests/test_upgrade_check_gate.py

TestTheSpeedStepGate.test_no_integrity_answer_scored_fails_it

1

0

1

benchmarks/tests/test_upgrade_check_gate.py

TestTheSpeedStepGate.test_without_the_gate_a_wrong_answer_is_not_read

1

0

1

tests/unit/benchmark/test_bench_energy.py

TestAvailability.test_on_windows_it_either_reads_rails_or_says_why

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestDiscovery.test_detects_a_real_model

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestDiscovery.test_backend_resolution_reaches_this_server

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestStreamingMetrics.test_stream_is_parsed_at_all

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestStreamingMetrics.test_ttft_is_measured_and_sane

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestStreamingMetrics.test_decode_rate_excludes_prefill

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestStreamingMetrics.test_usage_is_reported_not_estimated

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestStreamingMetrics.test_answer_time_is_recorded_only_for_a_finished_answer

0

1

1

tests/unit/benchmark/test_harness_against_ollama.py

TestCorrectnessProbe.test_probe_returns_a_scored_result

0

1

1

tests/unit/benchmark/test_v1_api.py

TestConnectivity.test_ollama_reachable

0

1

1

tests/unit/benchmark/test_v1_api.py

TestConnectivity.test_v1_reachable

0

1

1

tests/unit/benchmark/test_v1_api.py

TestListModels.test_returns_openai_format

0

1

1

tests/unit/benchmark/test_v1_api.py

TestListModels.test_model_structure

0

1

1

tests/unit/benchmark/test_v1_api.py

TestListModels.test_unknown_model_returns_404

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_basic_chat

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_usage_included

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_multiple_messages

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_streaming

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_temperature_and_top_p

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_missing_model_returns_error

0

1

1

tests/unit/benchmark/test_v1_api.py

TestChatCompletions.test_unknown_model_returns_error

0

1

1

tests/unit/benchmark/test_v1_api.py

TestCompletions.test_legacy_completion

0

1

1

tests/unit/benchmark/test_v1_api.py

TestCompletions.test_legacy_completion_streaming

0

1

1

tests/unit/benchmark/test_v1_api.py

TestEmbeddings.test_embedding

0

1

1

tests/unit/benchmark/test_v1_api.py

TestEmbeddings.test_embedding_batch

0

1

1

tests/unit/benchmark/test_v1_api.py

TestEmbeddings.test_embedding_usage

0

1

1

tests/unit/benchmark/test_v1_api.py

TestErrorHandling.test_empty_body

0

1

1

tests/unit/benchmark/test_v1_api.py

TestErrorHandling.test_invalid_json

0

1

1

tests/unit/benchmark/test_v1_api.py

TestErrorHandling.test_unsupported_endpoint

0

1

1

tests/unit/benchmark/test_v1_api.py

TestOllamaNativeApi.test_tags

0

1

1

tests/unit/benchmark/test_v1_api.py

TestOllamaNativeApi.test_version

0

1

1

tests/unit/test_thread_safety.py

test_the_modules_under_test_are_compiled_when_required

0

1

1

tests/unit/test_thread_safety.py

test_the_gil_stays_off_after_importing_every_module

0

1

1

benchmarks/tests/test_bench_coding_audit.py

TestCandidateConfinement.test_no_network_interfaces_when_the_kernel_allows_it

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestTheSixTasks.test_every_plausible_half_fix_is_rejected

0

12

12

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_correct_function_passes_and_names_the_analyzer

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_wrong_value_fails_with_partial_credit

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_assert_eq_tells_an_array_of_one_from_its_element

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_check_that_throws_is_a_failed_row_and_the_next_still_runs

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_failing_setup_statement_stops_the_checks

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_break_that_escapes_the_candidate_is_named

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_break_while_the_candidate_loads_is_named

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_a_solution_that_does_not_parse_is_named

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_rows_the_candidate_forged_are_refused

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_the_candidate_cannot_stand_in_for_the_helpers

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_the_checks_start_from_powershells_defaults

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_an_infinite_loop_is_killed

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_the_managed_heap_is_capped

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_an_allocating_candidate_is_stopped_at_the_cap

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestHarness.test_the_candidate_has_no_network

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestAnalyzer.test_an_error_severity_finding_fails_the_task

0

1

1

benchmarks/tests/test_bench_coding_powershell.py

TestAnalyzer.test_an_absent_analyzer_is_named_on_passing_and_failing_rows

0

1

1

TOTAL

2572

76

2648

Indices and Tables#